ai-research-survey

scan.json (25999B)

---

 {
   "paper": {
     "title": "Investigating Intersectional Bias in Large Language Models using Confidence Disparities in Coreference Resolution",
     "authors": [
       "Falaah Arif Khan",
       "Nivedha Sivakumar",
       "Yinong Oliver Wang",
       "Katherine Metcalf",
       "Cezanne Camacho",
       "Barry-John Theobald",
       "Luca Zappella",
       "Nicholas Apostoloff"
     ],
     "year": 2025,
     "venue": "COLM 2025",
     "arxiv_id": "2508.07111",
     "doi": "10.48550/arXiv.2508.07111"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLMs show coreference confidence disparities up to 40% across intersectional demographic attributes including body type, sexual orientation, and socio-economic status, with doubly-disadvantaged identities most affected in anti-stereotypical settings. Even hegemonic markers (e.g., 'White', 'cisgender') decrease coreference confidence compared to unaugmented baselines, suggesting models rely on memorization rather than reasoning. The WinoIdentity benchmark (245,700 prompts across 50 intersectional identities) reveals two independent failure modes—invalidity (memorization) and value misalignment (biased reasoning)—that can compound to cause social harm.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository explicitly linked: https://github.com/apple/ml-winoidentity (footnote 1, §3)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "WinoIdentity dataset released at the same GitHub link. The benchmark is constructed from the publicly available WinoBias dataset with documented augmentations."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of environment specifications, requirements files, or library versions in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The construction procedure is described conceptually but no runnable scripts or commands are detailed."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 2 and Figures 2-3 report point estimates of coreference confidence disparity with no confidence intervals or error bars. Table 3 reports mean ± std dev for coreference confidence but the main disparity metric lacks uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite claims of disparities between subgroups. Differences are reported as raw numbers without any hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as absolute confidence disparities with baseline context (e.g., '20 to 40%' disparities in Table 2), and specific coreference confidence values for each subgroup, allowing readers to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 1575 base sentences from WinoBias or 245,700 total prompts is sufficient. No power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports mean ± standard deviation for coreference confidence across prompts under different augmentation conditions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines included: no-augmentation baseline (gender-only disparity), non-demographic augmentation baseline (Table 2, §A.6), and comparison across 5 models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include Llama3-70b-instruct and Mixtral-8x7B, which are contemporary. The benchmark builds on the established WinoBias framework."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Three augmentation types (R-Aug, NR-Aug, C-Aug) are tested separately, along with non-demographic augmentation baselines (§A.6) and Chain-of-Thought prompting ablation (§A.5), effectively ablating the augmentation design."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Both coreference confidence (Eq. 1) and coreference accuracy are reported (Tables 3, 5), plus the disparity metric (Eq. 2) applied to both."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here; the benchmark has deterministic ground truth answers from WinoBias construction, and the study measures model confidence/accuracy against known correct coreferences."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates pre-trained models without training; there is no train/test split to manage. The entire WinoIdentity dataset is an evaluation corpus."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by demographic attribute (10 attributes), sentence type (Type-1 vs Type-2), augmentation type, individual occupations (Fig. 4), and per-model in Tables 2, 4, 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are discussed extensively: doubly-disadvantaged identities on anti-stereotypical occupations (§4.2, Fig. 4), models being confident and wrong on male-dominated occupations with feminine pronouns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The CoT prompting experiment (§A.5) shows that while CoT reduces disparities, it also decreases overall confidence, presenting a trade-off rather than a clean solution."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 40% confidence disparities are supported by Table 2 (e.g., socio-economic status at 0.4 for Mistral). Claims about doubly-disadvantaged identities are supported by §4.2 and Fig. 4. Memorization claims supported by §4.1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims models 'rely on memorization rather than reasoning' (§4.1, abstract) based on observational evidence (confidence drops with augmentation). This is a causal claim made from correlational evidence — the drop in confidence when adding identity markers could have explanations other than memorization, and no causal identification strategy is used."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly limits scope to the US context (§6 Ethical Considerations), acknowledges limitation to 50 identities and 1575 WinoBias sentences (§5 Limitations), and notes findings may not generalize to other cultural settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Non-demographic augmentation baseline (§A.6) tests whether any additional word (not just demographic markers) causes the effect. CoT experiment (§A.5) tests whether prompting strategy matters. The paper distinguishes memorization from biased reasoning effects."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes between what is measured (coreference confidence on synthetic prompts) and what it claims to indicate (representational harm in real-world decision contexts). §6 explicitly acknowledges that 'it's unclear how to translate mathematical unfairness into practical social harm.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions stated: mistral-7B-instruct-v0.2, mixtral-8x7B-instruct, llama3-70b-instruct, pythia-12B, falcon-40B-instruct (§4.1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt construction is described with examples (§3, Fig. 1), and an actual CoT prompt example is shown in §A.5. The prompts follow a deterministic template from WinoBias with clearly specified augmentations."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Greedy decoding is explicitly stated (§4, 'We use greedy decoding as this ensures deterministic predictions for reproducibility'), which specifies the key sampling parameter."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is direct model querying for next-token probabilities."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The dataset construction pipeline is thoroughly described (§3.2, Fig. 1): demographic markers from Table 1, three augmentation procedures, and how prompts are generated from WinoBias base sentences."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated 'Limitations and Future Work' subsection in §5 and a separate 'Ethical Considerations' section (§6) with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: limited to 50 identities and 1575 sentences (§5), combinatorial explosion problem (§5), US-specific cultural context (§6), difficulty translating mathematical unfairness to practical social harm (§6)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Paper explicitly states scope is limited to the US context (§6), binary gender intersected with 10 attributes (§3.2), coreference resolution task only (§5), and identifies specific expansions needed (other evaluation corpora, more markers)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The WinoIdentity dataset is released via GitHub (https://github.com/apple/ml-winoidentity), enabling independent reproduction and verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data construction is fully described: demographic markers drawn from the Wheel of Power and Privilege (Table 1), three augmentation procedures defined (§3.2), with the source WinoBias dataset clearly referenced."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data is synthetically constructed from WinoBias and demographic markers."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in §3.2 and Fig. 1: 1575 base WinoBias prompts × 2 pronouns × (25+1) demographic markers × 3 augmentation types = 245,700 prompts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors disclosed as affiliated with Apple (header: '1Work done while at Apple, 2Apple')."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Apple is not evaluating its own models. The evaluated models are Mistral, Mixtral, Llama3, Pythia, and Falcon — none are Apple products. Apple has no financial stake in the bias findings for these models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 5 evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses that WinoBias is likely in LLM training data and may be memorized (§1, §5: 'existing fairness benchmarks are likely to be included in LLM training data and potentially memorized'). This is central to the validity argument."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is a core theme. The paper argues WinoBias (2018) is likely memorized, and designs augmented prompts specifically to test whether models rely on memorization. The non-demographic augmentation baseline (§A.6) and the confidence drop for hegemonic markers both serve as contamination probes."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or computational requirements reported despite running 245,700 prompts across 5 models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, or hardware specifications reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Greedy decoding is used, making results deterministic (§4: 'We use greedy decoding as this ensures deterministic predictions for reproducibility'). No seed sensitivity because there is no randomness."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Single deterministic run per model due to greedy decoding — this is implicitly stated and appropriate since results are reproducible."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter tuning is performed. Models are evaluated off-the-shelf with greedy decoding."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection; all models evaluated with the same fixed setup."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons across 10 attributes, 5 models, 3 augmentations, and 2 sentence types, but no multiple comparison correction applied. No statistical tests performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party models, not their own system. No self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable; the study evaluates bias, not system performance improvement. Compute differences between models are not the focus."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses what WinoBias and WinoIdentity actually measure vs. what is claimed. The validity framework from Coston et al. (2023) is applied throughout (§1, §5), and the paper distinguishes validity concerns from value misalignment concerns."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; models are queried directly for next-token probabilities."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges WinoBias (2018) predates all evaluated models and is likely memorized (§1, §5). The augmentation framework is specifically designed to test beyond memorized patterns."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The augmentation design tests whether models are affected by information that should be irrelevant (demographic markers), effectively probing for a form of feature leakage from identity markers to coreference predictions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the WinoBias prompts are structurally independent or whether augmented variants of the same base prompt create non-independence issues in the analysis."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The augmentation framework itself serves as a leakage detection method: if models memorized WinoBias answers, augmented versions should show confidence drops, which is exactly what is observed and analyzed (§4.1)."
       }
     }
   },
   "claims": [
     {
       "claim": "Coreference confidence disparities reach as high as 40% along demographic attributes including body type, sexual orientation, and socio-economic status.",
       "evidence": "Table 2 shows Mistral has 0.4 disparity for socio-economic status (Type-2), 0.392 for body type (Type-2), and multiple models exceed 0.2 across attributes. Table 4 extends this to all augmentation types.",
       "supported": "strong"
     },
     {
       "claim": "Models are most uncertain about doubly-disadvantaged identities in anti-stereotypical settings.",
       "evidence": "§4.2 and Fig. 4 show that for mechanic, fem coreference confidence is -0.065, transgender fem is -0.11, and gay fem is -0.24, demonstrating compounding disadvantage.",
       "supported": "strong"
     },
     {
       "claim": "LLMs rely more on memorization than reasoning, as evidenced by confidence drops even for hegemonic markers.",
       "evidence": "§4.1, Figs. 9-10 show coreference confidence consistently decreases after referent augmentation even for hegemonic markers like 'White' and 'cisgender'. The non-demographic baseline (§A.6) also shows drops but to a lesser extent.",
       "supported": "moderate"
     },
     {
       "claim": "Uncertainty-based evaluation detects bias that accuracy-based evaluation misses.",
       "evidence": "§A.3.1 and Table 3 show accuracy increases under non-referent augmentation (suggesting improvement) while confidence analysis reveals this is due to bias (models less likely to pick the augmented non-referent), correctly identifying the mechanism.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-Thought prompting reduces intersectional disparities but at the cost of overall confidence.",
       "evidence": "Tables 6-9 (§A.5) show CoT generally reduces disparities but also decreases coreference confidence for both Type-1 and Type-2 sentences across most attributes.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "Despite comparing disparities across 10 demographic attributes, 5 models, 3 augmentation types, and 2 sentence types, no statistical significance tests are performed. All comparisons are based on raw numerical differences, making it unclear which observed disparities are statistically meaningful versus noise."
     },
     {
       "flag": "Causal claims from observational evidence",
       "detail": "The memorization claim is central to the paper's narrative but is inferred from confidence drops upon augmentation. Alternative explanations (distributional shift, syntactic complexity from added tokens) are partially but not fully addressed."
     },
     {
       "flag": "Company evaluating fairness of competitors' models",
       "detail": "Apple researchers evaluate Mistral, Meta (Llama), and other companies' models for bias. While the evaluated models are not Apple products (reducing direct conflict), the finding that competitors' models are biased could serve Apple's interests."
     }
   ],
   "cited_papers": [
     {
       "title": "On the dangers of stochastic parrots: Can language models be too big?",
       "authors": ["Emily M Bender", "Timnit Gebru", "Angelina McMillan-Major", "Shmargaret Shmitchell"],
       "year": 2021,
       "relevance": "Foundational work on risks of large language models, relevant to AI safety and alignment research."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenges emergent abilities narrative in LLMs, relevant to LLM capability evaluation methodology."
     },
     {
       "title": "GSM-Symbolic: Understanding the limitations of mathematical reasoning in large language models",
       "authors": ["Iman Mirzadeh", "Keivan Alizadeh", "Hooman Shahrokhi"],
       "year": 2024,
       "arxiv_id": "2410.05229",
       "relevance": "Demonstrates LLM reasoning limitations through symbolic math tasks, directly relevant to AI capability assessment."
     },
     {
       "title": "Gender bias in coreference resolution: Evaluation and debiasing methods",
       "authors": ["Jieyu Zhao", "Tianlu Wang", "Mark Yatskar", "Vicente Ordonez", "Kai-Wei Chang"],
       "year": 2018,
       "relevance": "Original WinoBias paper that this work extends; foundational benchmark for bias evaluation in NLP."
     },
     {
       "title": "A validity perspective on evaluating the justified use of data-driven decision-making algorithms",
       "authors": ["Amanda Coston", "Anna Kawakami", "Haiyi Zhu", "Ken Holstein", "Hoda Heidari"],
       "year": 2023,
       "relevance": "Validity framework applied in this paper to distinguish validity concerns from value alignment in AI systems."
     },
     {
       "title": "Uncertainty as a fairness measure",
       "authors": ["Selim Kuzucu", "Jiaee Cheong", "Hatice Gunes", "Sinan Kalkan"],
       "year": 2024,
       "relevance": "Formalizes uncertainty-based fairness evaluation, directly inspiring the methodology of this paper."
     },
     {
       "title": "Gender, race, and intersectional bias in resume screening via language model retrieval",
       "authors": ["Kyra Wilson", "Aylin Caliskan"],
       "year": 2024,
       "relevance": "Studies intersectional bias in LLM-based hiring, directly relevant to AI fairness and LLM evaluation."
     },
     {
       "title": "BBQ: A hand-built bias benchmark for question answering",
       "authors": ["Alicia Parrish", "Angelica Chen", "Nikita Nangia"],
       "year": 2022,
       "relevance": "Major bias benchmark for LLM evaluation across multiple demographic attributes."
     },
     {
       "title": "The silicone ceiling: Auditing GPT's race and gender biases in hiring",
       "authors": ["Lena Armstrong", "Abbey Liu", "Stephen MacNeil", "Danaë Metaxa"],
       "year": 2024,
       "arxiv_id": "2405.04412",
       "relevance": "Audits GPT for hiring bias, relevant to LLM fairness evaluation methodology."
     },
     {
       "title": "A survey on fairness in large language models",
       "authors": ["Yingji Li", "Mengnan Du", "Rui Song", "Xin Wang", "Ying Wang"],
       "year": 2023,
       "arxiv_id": "2308.10149",
       "relevance": "Comprehensive survey of LLM fairness evaluations, relevant as meta-research on AI evaluation methodology."
     }
   ]
 }