ai-research-survey

scan.json (28137B)

---

 {
   "paper": {
     "title": "The African Woman is Rhythmic and Soulful: An Investigation of Implicit Biases in LLM Open-ended Text Generation",
     "authors": [
       "Serene Lim",
       "María Pérez-Ortiz"
     ],
     "year": 2024,
     "venue": "University College London (preprint/workshop paper)"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, repository URL, or code archive is provided. The paper mentions experimental data was stored on a secure database linked to the OpenAI API, but no code is released for reproduction."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The paper states experimental data was recorded on a secure database, but no download link or public archive is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification is provided. The paper only mentions 'GPT-4 via the interface of ChatGPT' with no library versions, API version details, or dependency files."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. While the methodology describes the experimental setup in narrative form, there are no scripts or README-style instructions that would allow replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Results are stated as raw counts or percentages (e.g., '9 of 10 trials', '80% of responses') with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The paper makes comparative claims (e.g., the LLM IAT correlates with traditional methods) based solely on observed counts with no formal testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Effect sizes are not reported. The paper provides raw counts and qualitative observations but no Cohen's d, odds ratios, or other standardized effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 10 trials per experiment is not justified. No power analysis or discussion of whether 10 repetitions is sufficient to support the claims is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Variance is not reported. Results are presented as single counts or percentages across 10 trials with no standard deviation or spread measure (e.g., '9 of 10 trials' but no variance across runs)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are compared against. The paper reports GPT-4's outputs but does not compare against other LLMs, prior bias measurement tools, or any baseline condition to contextualize findings."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper could and should have included contemporary baselines (other LLMs, prior bias detection methods). The absence of any baselines means this criterion applies but is not satisfied. The scan-agent instructions specify that applies=false is for structurally inapplicable criteria, not for when a paper fails to include something it should have."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper tests five different experimental tasks but does not systematically isolate variables to determine which components drive bias manifestation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper employs multiple assessment approaches: LLM IAT (word association counts), Decision Bias (qualitative output analysis), Sycophancy, Word Generation (word clouds), and Story Generation (thematic analysis), constituting multiple complementary metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The authors informally analyzed GPT-4 outputs through thematic observation, but this does not constitute a proper human evaluation study. There is no systematic coding scheme, no inter-rater reliability, no evaluation rubric, and no independent evaluators. The schema requires 'human ratings, manual inspection, user studies, expert review' — the paper's informal qualitative observations lack the methodological rigor to qualify."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is not a machine learning training study with dev/test splits. The paper uses LLM API calls with manually designed prompts; there is no train/test distinction."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported separately for each of five experimental tasks (LLM IAT, Decision Bias, Sycophancy, Word Generation, Story Generation) and broken down by attribute type (gender-career, race-valence, Muslim-others IAT)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses the model's rejection of the visual IAT ('90% of the time' it uses reject option classification for images with sensitive characteristics) as a notable failure of the approach, and notes limitations of the Decision Bias task with race-only prompts."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the race-only Decision Bias results 'did not show much difference' and that adding race as a variable into prompts did not substantially change outcomes beyond gender, constituting a negative/null result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the LLM IAT Bias 'correlates with traditional methods' but the paper does not demonstrate a statistical correlation — it only shows qualitative consistency. The claim of providing 'a more comprehensive framework' is not empirically validated against prior frameworks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims (e.g., 'LLMs trained on biased data perpetuate biases', 'RLHF causes sycophancy') but the study design is purely observational — all experiments are prompt-and-observe with no controlled manipulation of training data or causal identification strategy."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Claims are framed broadly about 'LLMs' in general but only GPT-4 via ChatGPT interface is tested. The paper draws conclusions about 'AI systems' and 'LLMs' broadly without bounding claims to GPT-4 specifically."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive alternative explanations for results are discussed. The paper does not consider confounds such as prompt phrasing effects, temperature settings, or whether results reflect RLHF guardrails versus actual model weights. The mention that race may have been subordinated to gender is a brief observation, not an analysis of alternative explanations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states only 'GPT-4 via the interface of ChatGPT' without specifying a version number, API snapshot date, or model identifier. 'GPT-4' without a version does not meet the criterion."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Actual prompt text is provided verbatim for the key experiments: the LLM IAT prompt (word list with name assignment instruction), the Decision Bias prompt ('Generate two short profiles, within 100 words, about Rebecca and John...'), and the Word Generation and Story Generation prompts are quoted directly in the methodology."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported. The paper uses ChatGPT as an interface without disclosing API settings that would significantly affect output variability."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper accesses GPT-4 through the ChatGPT interface with direct prompts; there is no agentic loop, tool use, or multi-step scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing documentation is provided. The paper goes from collecting LLM outputs directly to reporting results without describing any filtering, coding scheme, or inter-rater reliability procedure for the qualitative analysis."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper has an Ethics section but it only states no privacy violations occurred; a substantive limitations discussion is absent."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. The scope of generalizability to other LLMs, the small number of trials (n=10), or the absence of inter-rater reliability for qualitative coding are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Scope boundaries are not explicitly stated. The paper does not specify that results apply only to GPT-4, only to the tested prompts, or only to the English language — it generalizes broadly to 'LLMs' without stating what it does NOT claim."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (actual LLM response transcripts) is not released. The Ethics section mentions data is stored on a secure database but no access is provided to third parties for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: experiments were conducted via ChatGPT GPT-4 interface, each task repeated 10 times, with specific prompts provided. The methodology section describes what data was collected for each of the five experiment types."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — there are no human participants. The study collects LLM outputs rather than recruiting human subjects; the data source is the GPT-4 API."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw LLM outputs to reported findings is not documented. It is unclear how qualitative themes were identified, whether multiple coders were used, and how word clouds were generated from the outputs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": false,
         "answer": false,
         "justification": "The paper appears to be unfunded student work at UCL (author email format 'serene.lim.21@ucl.ac.uk' suggests student enrollment). No acknowledgments section, no grant numbers, and no funding sources are mentioned. Per the schema, NA applies for 'clearly unfunded work.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors' affiliations are clearly stated as University College London, London, United Kingdom in the paper header. Neither author appears to be affiliated with OpenAI whose product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so this criterion cannot be evaluated. The paper appears to be unfunded academic work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears in the paper. There is no disclosure of patents, equity holdings, or other financial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this paper does not evaluate GPT-4's capability on any benchmark. The study uses novel author-designed prompts to probe implicit biases, not pre-existing benchmark tasks. The schema says NA 'if the paper does not evaluate a pre-trained model's capability on any benchmark.'"
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is not a benchmark evaluation study where train/test overlap is a concern. The paper uses original prompts constructed by the authors rather than evaluating on a fixed public benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study does not use a pre-existing benchmark dataset. The IAT stimuli and prompts are adapted from psychological literature but not from a fixed ML benchmark corpus."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved. The study evaluates LLM outputs, not human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved. The Ethics section confirms this: 'This work uses language models... will not have any implications on any other participants, nor will any privacy guidelines be breached.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved. Criteria would apply to participant recruitment, which is absent from this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants or experimental groups. The paper mentions 'randomizing the order of stimuli' but this is in LLM prompting, not human participant randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved, so attrition is not relevant."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or API cost is reported. The paper uses the ChatGPT interface for GPT-4 queries but provides no estimate of tokens consumed or API costs incurred."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget is stated. The total number of API calls (5 experiments × 10 trials each, with variations) is not totaled, and no cost estimate is provided."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 demonstrates strong implicit gender-career biases in the LLM IAT, associating female names with family words and male names with career words in 9 of 10 trials with 100% accuracy on both categories.",
       "evidence": "LLM IAT results: '7 of 7 family related words were assigned to the female name, and 7 of 7 career related words were assigned to the male name in 9 of 10 trials.' Verbatim LLM output is provided in Results section.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 demonstrates strong racial bias in the race-valence IAT, assigning 8 of 8 positive words to white and 8 of 8 negative words to black names in 10 of 10 experiments.",
       "evidence": "Results section states: 'word associations between white-black and valence (good-bad) were similarly biased, with 8 of 8 positive words being assigned to white, and 8 of 8 negative words being assigned to black in 10 of 10 experiments.'",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 exhibits sycophantic behavior, adjusting its responses to reduce bias after a user points out bias, suggesting RLHF training makes the model hypersensitive to user feedback rather than intrinsically unbiased.",
       "evidence": "Sycophancy results section documents that after asking 'Why?' the model recognized bias and changed subsequent responses: 'Had the Why? question not been asked, there would have been no difference between the results of the female versus male names.'",
       "supported": "weak"
     },
     {
       "claim": "The LLM IAT Bias correlates with traditional bias measurement methods and more effectively predicts downstream behaviors as measured by the LLM Decision Bias.",
       "evidence": "Claimed in the abstract, but no quantitative correlation analysis is presented. The support is purely qualitative: both IAT and Decision Bias show gender stereotyping, which the authors interpret as correlation.",
       "supported": "weak"
     },
     {
       "claim": "Story generation exhibits culturally stereotypical patterns including exoticization of racial attributes (e.g., Chinese prompts produce stories about dragons and dynasties) and gender-differentiated narrative structures.",
       "evidence": "Story Generation results section provides thematic analysis with specific examples: 'mentioning Chinese resulted in narratives heavily centred around Chinese dynasties, dragons, and traditional paintings.' Female characters received mentor figures while male characters were depicted as independent.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "observational",
     "benchmark-eval"
   ],
   "key_findings": "GPT-4 exhibits strong implicit gender-career and race-valence biases when tested via a modified Implicit Association Test (IAT), even after safety training — assigning 100% of family words to female names and career words to male names in 9/10 trials, and 100% of positive/negative words to white/black names in 10/10 trials. The model shows sycophantic behavior, reducing bias expression after user feedback, which the authors interpret as context-sensitivity rather than genuine debiasing. Qualitative analysis of word generation and story outputs reveals persistent gender stereotyping (females as nurturing, males as leaders) and racial exoticization (African = rhythmic/soulful, Chinese = dragons/dynasties). The paper argues these findings demonstrate that bias in LLMs cannot be eliminated through optimization alone and requires sociotechnical approaches informed by digital anthropology and feminist STS.",
   "red_flags": [
     {
       "flag": "Extremely small sample (n=10 trials)",
       "detail": "All experiments are repeated only 10 times with no justification for this sample size, no confidence intervals, and no statistical tests. The strong results (9/10 or 10/10 consistency) may be reliable, but the small n and lack of statistical analysis make it impossible to assess whether less extreme effects are meaningful."
     },
     {
       "flag": "Single model, no comparison",
       "detail": "All experiments are conducted only on GPT-4 via the ChatGPT interface. No other LLMs are tested, making it impossible to determine whether findings are GPT-4-specific or general to LLMs. The abstract and title make broad claims about 'LLMs'."
     },
     {
       "flag": "No inter-rater reliability for qualitative coding",
       "detail": "The thematic analysis of story generation and word clouds is conducted by the authors without describing a coding scheme, inter-rater reliability, or blind review process. Qualitative claims about bias themes are not independently validated."
     },
     {
       "flag": "Unspecified model version and API parameters",
       "detail": "The paper uses 'GPT-4 via the interface of ChatGPT' without specifying a snapshot version or API parameters (temperature, top-p). GPT-4 behavior varies significantly across versions and settings, making replication impossible and results non-transferable."
     },
     {
       "flag": "Abstract claim not empirically supported",
       "detail": "The abstract claims the LLM IAT Bias 'correlates with traditional methods and more effectively predicts downstream behaviors' but no quantitative correlation analysis is presented — the claim rests on qualitative observation that both approaches show stereotyping."
     },
     {
       "flag": "Overgeneralization from narrow test",
       "detail": "The paper draws broad conclusions about 'LLMs' and their social harms from a small set of prompts testing two names (Rebecca/John) across a handful of word lists. The scope of bias types tested is narrow (gender-career, race-valence, Muslim-valence) relative to the breadth of the claims made."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring Implicit Bias in Explicitly Unbiased Large Language Models",
       "authors": [
         "Bai, X.",
         "Wang, A.",
         "Sucholutsky, I.",
         "Griffiths, T."
       ],
       "year": 2024,
       "arxiv_id": "2402.04105",
       "relevance": "Directly related work on measuring implicit bias in LLMs using IAT-inspired methods; the paper under review builds on this methodology."
     },
     {
       "title": "On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?",
       "authors": [
         "Bender, E.",
         "McMillan-Major, A.",
         "Shmitchell, S.",
         "Gebru, T."
       ],
       "year": 2021,
       "doi": "10.1145/3442188.3445922",
       "relevance": "Foundational paper on bias and harms in large language models; directly cited as theoretical background for bias in training data."
     },
     {
       "title": "Semantics derived automatically from language corpora contain human-like biases",
       "authors": [
         "Caliskan, A.",
         "Bryson, J.J.",
         "Narayanan, A."
       ],
       "year": 2017,
       "doi": "10.1126/science.aal4230",
       "relevance": "Foundational work on Word Embedding Association Tests (WEAT) for measuring bias in language models; the LLM IAT methodology builds on this."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": [
         "Brown, T.",
         "Mann, B.",
         "Ryder, N."
       ],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "GPT-3 paper; foundational work on few-shot learning in LLMs directly relevant to the few-shot learning experiment conducted."
     },
     {
       "title": "Language (Technology) is Power: A Critical Survey of 'Bias' in NLP",
       "authors": [
         "Blodgett, S.",
         "Barocas, S.",
         "Iii, H.",
         "Wallach, H."
       ],
       "year": 2020,
       "arxiv_id": "2005.14050",
       "relevance": "Critical survey of bias in NLP directly relevant to the paper's theoretical framing and methodology evaluation."
     },
     {
       "title": "'Kelly is a Warm Person, Joseph is a Role Model': Gender Biases in LLM-Generated Reference Letters",
       "authors": [
         "Wan, Y.",
         "Pu, G.",
         "Sun, J.",
         "Garimella, A.",
         "Chang, K.-W.",
         "Peng, N."
       ],
       "year": 2023,
       "doi": "10.48550/arXiv.2310.09219",
       "relevance": "Related work on gender bias in LLM open-ended generation; cited for finding that women receive warmer adjectives and men receive leadership descriptors in LLM-generated text."
     },
     {
       "title": "Challenging systematic prejudices: an investigation into bias against women and girls in large language models",
       "authors": [
         "van Niekerk, D.",
         "Pérez-Ortiz, M.",
         "Shawe-Taylor, J.",
         "Orlič, D.",
         "Drobnjak, I.",
         "Kay, J."
       ],
       "year": 2024,
       "relevance": "UNESCO report using word cloud visualizations to study gender bias in LLMs; directly related work that uses the same word generation analysis methodology."
     },
     {
       "title": "BiasAsker: Measuring the Bias in Conversational AI System",
       "authors": [
         "Wan, Y.",
         "Wang, W.",
         "He, P.",
         "Gu, J.",
         "Bai, H.",
         "Lyu, M."
       ],
       "year": 2023,
       "arxiv_id": "2305.12434",
       "relevance": "Related work on measuring bias in conversational AI systems; directly relevant to the methodology and scope of this paper."
     },
     {
       "title": "The Woman Worked as a Babysitter: On Biases in Language Generation",
       "authors": [
         "Sheng, E.",
         "Chang, K.-W.",
         "Natarajan, P.",
         "Peng, N."
       ],
       "year": 2019,
       "arxiv_id": "1909.01326",
       "relevance": "Early work on gender bias in language generation relevant to the paper's story generation experiments."
     },
     {
       "title": "A Survey on Fairness in Large Language Models",
       "authors": [
         "Li, Y.",
         "Du, M.",
         "Song, R.",
         "Wang, X.",
         "Wang, Y."
       ],
       "year": 2024,
       "doi": "10.48550/arXiv.2308.10149",
       "relevance": "Survey of fairness approaches in LLMs, directly relevant to the paper's comparative analysis of bias mitigation techniques."
     },
     {
       "title": "Lipstick on a Pig: Debiasing Methods Cover up Systematic Gender Biases in Word Embeddings But do not Remove Them",
       "authors": [
         "Gonen, H.",
         "Goldberg, Y."
       ],
       "year": 2019,
       "arxiv_id": "1903.03862",
       "relevance": "Cited to support the claim that debiasing methods may produce 'reverse discrimination' or superficial fixes rather than genuine bias removal."
     },
     {
       "title": "Co-Writing with Opinionated Language Models Affects Users' Views",
       "authors": [
         "Jakesch, M.",
         "Bhat, A.",
         "Buschek, D.",
         "Zalmanson, L.",
         "Naaman, M."
       ],
       "year": 2023,
       "doi": "10.1145/3544548.3581196",
       "relevance": "Relevant to the sycophancy finding — examines how interaction with opinionated LLMs affects human beliefs, directly related to the paper's theme of LLM influence on societal norms."
     }
   ]
 }