ai-research-survey

scan-v5.json (28274B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Investigating Intersectional Bias in Large Language Models using Confidence Disparities in Coreference Resolution",
     "authors": [
       "Falaah Arif Khan",
       "Nivedha Sivakumar",
       "Yinong Oliver Wang",
       "Katherine Metcalf",
       "Cezanne Camacho",
       "Barry-John Theobald",
       "Luca Zappella",
       "Nicholas Apostoloff"
     ],
     "year": 2025,
     "venue": "COLM 2025",
     "arxiv_id": "2508.07111",
     "doi": "10.48550/arXiv.2508.07111"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All quantitative claims in the abstract (245,700 prompts, disparities up to 40%, doubly-disadvantaged identities worst off, hegemonic marker drops) are backed by Tables 2–5 and Figs. 2–10 in the paper.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims LLMs rely on 'memorization rather than logical reasoning' and that augmentations 'trigger and reveal bias,' but the observational design only shows correlation/disparity patterns; no controlled causal experiments (e.g., training data ablations or probing) are performed.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Abstract and conclusion make broad claims about 'LLMs' and their use 'in hiring and admissions,' but only five specific open-weight models are tested on one coreference corpus (WinoBias derivatives); closed-source and larger frontier models are excluded with no qualification of this scope gap.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes a non-demographic augmentation baseline to distinguish demographic-specific brittleness from general brittleness (App. A.6), and explicitly contrasts memorization vs. reasoning as alternative explanations for performance drops.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines coreference confidence (Eq. 1) as next-token log-probability differences, frames this as a proxy for fairness under uncertainty, and distinguishes it from accuracy-based measures, citing Kuzucu et al. (2024) for theoretical grounding.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 contains a dedicated 'Limitations and Future Work' paragraph listing scope restrictions, and Section 6 'Ethical Considerations' adds further limitations.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific limitations named: restriction to 50 identities and 1,575 WinoBias sentences, combinatorial explosion in evaluation, US-centric demographic markers, and inability to translate mathematical unfairness to practical social harm threshold.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'Our research is specifically focused on the US context, and therefore, generalizability of our findings to other cultural and sociolinguistic settings is uncertain and may be limited.'",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or disclosure section exists in the paper; only author affiliations are noted.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All eight authors are disclosed as Apple employees or as doing work at Apple; affiliations are stated on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates five external open-weight models (Mistral, Mixtral, Llama3, Pythia, Falcon) rather than Apple's own products, so the funder (Apple) is not directly advantaged by the findings.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Coreference confidence (Eq. 1), coreference confidence disparity (Eq. 2), intersectional bias, hegemonic/disadvantaged identities, Type-1/Type-2 sentences, and augmentation types (R-Aug, NR-Aug, C-Aug) are all formally defined.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly states two contributions: (1) the WinoIdentity benchmark and flexible construction framework, and (2) empirical evaluation findings raising validity and value-misalignment concerns.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 distinguishes this work from 15+ prior benchmarks on single-axis vs. intersectional fairness, uncertainty-based vs. accuracy-based evaluation, and from contemporaneous work by Kotek et al. (2023), explaining how this paper extends and differs from each.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "A GitHub URL (https://github.com/apple/ml-winoidentity) is provided in Section 1 footnote 1.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "WinoIdentity is released via the same GitHub repository; the paper also relies on WinoBias, a publicly available benchmark.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or dependency list is mentioned in the paper; only greedy decoding is specified as an implementation choice.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The augmentation procedure is described mathematically but no step-by-step instructions for reproducing experiments (model loading, tokenization, evaluation loop) are provided in the paper.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Table 2 (the primary result table showing disparities up to 40%) reports only point estimates; Table 3 reports mean ± std for average confidence but not for the key disparity metric.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical hypothesis tests are applied anywhere in the paper; results are reported as raw averages and maximum disparities without p-values or confidence intervals.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Disparities are reported as raw differences on a [0,1] scale (e.g., 0.40 for socio-economic status in Mistral), providing an interpretable effect size relative to the possible range.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 1,575 base sentences are taken directly from WinoBias with no power analysis or sample size justification; the 25 demographic markers are chosen based on The Wheel of Power and Privilege without statistical rationale.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 reports mean ± std for coreference confidence under each augmentation condition (e.g., '0.322 ± 0.693'), providing variance information for absolute confidence scores.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Both a 'no augmentation' baseline (original WinoBias) and a 'non-demographic augmentation' baseline (neutral adjectives like 'confused,' 'relaxed') are included to contextualize disparity results.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Five models are compared: Mistral-7B-instruct-v0.2, Mixtral-8x7B, Llama3-70B, Pythia-12B, and Falcon-40B, all released 2022–2024 and representing the contemporary open-weight landscape.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Three augmentation types (R-Aug, NR-Aug, C-Aug) serve as a structured ablation isolating where demographic information is placed; CoT vs. no-CoT for Mistral (App. A.5) is an additional ablation.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Both coreference confidence disparity (uncertainty-based) and accuracy disparity (Table 5) are reported, enabling comparison between the two evaluation approaches.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "The study is a fully automated benchmark evaluation; human judgment of model outputs is not part of the methodology.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "This is an evaluation study, not a training/prediction task; the WinoIdentity corpus IS the test set and no train/test split is needed.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by demographic attribute (10 categories), model (5), sentence type (Type-1/Type-2), augmentation type (3), and referent occupation (Figs. 4, 11–14), providing extensive granularity.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.2 explicitly analyzes failure cases: fem:mechanic avg confidence -0.065 vs. transgender:fem -0.11 and gay:fem -0.24, quantifying double-disadvantage effects per occupation.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "App. A.5 reports that CoT prompting reduces overall confidence while only partially reducing disparities — a negative result for CoT as a mitigation strategy — presented transparently.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact model identifiers are provided: mistral-7B-instruct-v0.2, mixtral-8x7B-instruct, llama3-70b-instruct, pythia-12B, falcon-40B-instruct.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 shows example prompts, Section 4 describes the exact format ('The pronoun [pronoun] refers to the'), and App. A.5 provides a full multi-turn CoT prompt example.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Greedy decoding is explicitly specified ('We use greedy decoding as this ensures deterministic predictions for reproducibility'); no sampling hyperparameters apply.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; models are queried directly via next-token probability extraction.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full augmentation procedure is documented in Section 3.2 and Fig. 1, including the formula for augmented set size (2(|G|+1)) and how contrastive markers are handled for multi-value attributes.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "The WinoIdentity dataset (245,700 prompts) is released at the GitHub URL provided; raw benchmark data is publicly accessible.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 fully describes the data construction pipeline: source corpus (WinoBias), demographic markers (Table 1, The Wheel of Power and Privilege), augmentation logic, and resulting corpus size.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants are involved; the benchmark is constructed programmatically from existing corpora.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The complete pipeline from WinoBias base sentences through augmentation types (R-Aug, NR-Aug, C-Aug) to the final 245,700 prompts is documented in Section 3 with formal notation.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoffs are stated for any of the five evaluated models; the contamination concern is raised motivationally but not addressed with specific cutoff dates.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5 explicitly raises that 'existing fairness benchmarks are likely to be included in LLM training data and potentially memorized,' which directly motivates the augmentation framework as a contamination mitigation strategy.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly designs WinoIdentity as a contamination-resistant extension of WinoBias, arguing that the augmented prompts are unlikely to appear verbatim in training corpora.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; not applicable.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; not applicable.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; not applicable.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; not applicable.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; not applicable.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; not applicable.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; not applicable.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference costs or latency figures are reported for running 245,700 prompts across five models.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total compute budget, GPU hours, or hardware specifications are disclosed.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs exhibit coreference confidence disparities as high as 40% across intersectional demographic groups, with the largest disparities for socio-economic status, body type, disability, and sexual orientation.",
       "evidence": "Table 2 shows Mistral disparities of 0.389 (Type-1) and 0.400 (Type-2) for socio-economic status, 0.392 for body type (Type-2); Table 4 extends these findings across all augmentation types.",
       "supported": "strong"
     },
     {
       "claim": "Doubly-disadvantaged intersectional identities (e.g., gay or transgender women) in anti-stereotypical occupational settings (e.g., mechanic) show exacerbated underconfidence compared to single-axis disadvantaged groups.",
       "evidence": "Section 4.2 reports: fem:mechanic avg confidence -0.065, transgender:fem:mechanic -0.11, gay:fem:mechanic -0.24 for Mistral C-Aug conditions.",
       "supported": "moderate"
     },
     {
       "claim": "Coreference confidence decreases even for hegemonic/privileged demographic markers (White, cisgender, heterosexual) after referent augmentation, suggesting LLMs rely on memorization rather than genuine reasoning.",
       "evidence": "Figs. 9 and 10 (App. A.2) show confidence decreases for all 10 demographic attributes including hegemonic markers for all models except Pythia, which was already near-zero confidence.",
       "supported": "moderate"
     },
     {
       "claim": "Less performant models (Pythia) show smaller confidence disparities than more performant models (Mistral), revealing a performance-fairness tradeoff.",
       "evidence": "Table 2 shows Pythia disparities below 0.12 in all demographic attributes vs. Mistral disparities of 0.20–0.40 in 7 of 10 attributes; this tradeoff is explicitly discussed in Section 4.1.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-thought prompting reduces confidence disparities but also decreases overall coreference confidence, trading parity for accuracy.",
       "evidence": "Tables 6–9 (App. A.5) show that CoT generally halves disparities (e.g., ses Type-2: 0.400→0.216) but also substantially reduces mean confidence (e.g., no-aug Type-2 fem: 0.600→0.480).",
       "supported": "moderate"
     },
     {
       "claim": "Uncertainty-based evaluation detects bias that accuracy-based measures obscure, as models can improve accuracy by becoming more biased toward non-referent exclusion.",
       "evidence": "App. A.3.1 (Table 3) shows accuracy increases under non-referent augmentation for all models except Llama3, while coreference confidence decreases on Type-2 sentences — the spurious accuracy gain masks bias.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "The paper introduces WinoIdentity, a 245,700-prompt intersectional fairness benchmark built by augmenting WinoBias with 25 demographic markers across 10 attributes intersected with binary gender. Evaluating five open-weight LLMs, confidence disparities exceed 20% in 7 of 10 demographic attributes, reaching 40% for socio-economic status and body type, with doubly-disadvantaged identities (e.g., gay or transgender women assigned to male-dominated occupations) showing the most severe underconfidence. Hegemonic markers (White, cisgender) also cause confidence drops — not just disadvantaged ones — suggesting models rely on memorization rather than intersectional reasoning. Two independent failure modes are identified: validity failure (poor performance on augmented prompts indicating memorization) and value misalignment (disparate confidence across demographic groups), which can compound to amplify real-world identity-based harms.",
   "red_flags": [
     {
       "flag": "Memorization claim unsupported by design",
       "detail": "The conclusion that LLMs 'rely on memorization rather than reasoning' is inferred from performance degradation under augmentation, but no direct test of memorization (training data membership inference, probing, or causal intervention) is performed."
     },
     {
       "flag": "Frontier models excluded",
       "detail": "Only five older open-weight models are evaluated; no closed-source or frontier models (GPT-4, Claude, Gemini) appear despite the paper claiming broad implications for LLM deployment in hiring and admissions."
     },
     {
       "flag": "Binary gender only",
       "detail": "Despite focusing on intersectional identity, the gender axis uses only binary masculine/feminine pronouns, excluding non-binary identities that are explicitly mentioned as vulnerable in related work (Hossain et al., 2023)."
     },
     {
       "flag": "No significance testing on main results",
       "detail": "All disparity comparisons in Tables 2, 4, 5 are point estimates without confidence intervals or hypothesis tests; it is unclear whether observed differences exceed measurement noise, especially for smaller disparities."
     },
     {
       "flag": "Overgeneralization from 5 models to 'LLMs'",
       "detail": "Conclusions frame findings as properties of 'LLMs' generally, but coverage is limited to five specific open-weight models from 2022–2024, with no scope qualification in the abstract or conclusion."
     }
   ],
   "cited_papers": [
     {
       "title": "Gender bias in coreference resolution: Evaluation and debiasing methods (WinoBias)",
       "relevance": "Foundation corpus extended by this work; coreference resolution bias benchmark this study is built upon."
     },
     {
       "title": "Uncertainty as a fairness measure",
       "relevance": "Provides theoretical grounding for uncertainty-based fairness evaluation, which the paper's core metric (coreference confidence disparity) is based on."
     },
     {
       "title": "A validity perspective on evaluating the justified use of data-driven decision-making algorithms",
       "relevance": "Framework distinguishing validity from value alignment that structures the paper's two-failure-mode analysis."
     },
     {
       "title": "Gender bias and stereotypes in large language models (Kotek et al., 2023)",
       "relevance": "Contemporaneous work extending WinoBias to evaluate reasoning strategies; paper explicitly contrasts its uncertainty-based approach against this accuracy-based one."
     },
     {
       "title": "Bias out-of-the-box: an empirical analysis of intersectional occupational biases in popular generative language models (Kirk et al., 2024)",
       "relevance": "Prior work on intersectional occupational bias in LLMs that this paper builds upon and extends to uncertainty-based evaluation."
     },
     {
       "title": "Intersectional stereotypes in large language models: Dataset and analysis (Ma et al., 2023)",
       "relevance": "Related intersectional stereotype dataset using 6 demographic attributes; compared in related work as prior intersectionality benchmark."
     },
     {
       "title": "Gender, race, and intersectional bias in resume screening via language model retrieval (Wilson & Caliskan, 2024)",
       "relevance": "Directly evaluates downstream allocational harms (resume screening) that motivate this paper's focus on LLM validity and fairness."
     },
     {
       "title": "Factoring the matrix of domination: A critical review and reimagination of intersectionality in AI fairness (Ovalle et al., 2023)",
       "relevance": "Comprehensive review of intersectionality in fair-ML cited as foundational context for the intersectional framing."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly addresses LLM bias in high-stakes hiring/admissions contexts with a released benchmark, but provides no practical mitigation strategies beyond noting CoT has limited effectiveness."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that hegemonic/privileged markers (White, cisgender) also cause confidence drops contradicts the expected 'unmarked' behavior and challenges simple bias narratives."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Explicitly raises concerns about real-world discrimination in hiring (e.g., down-ranking resumes mentioning 'Black Feminist Scholars' or 'Neurodivergent in AI Affinity Group')."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Standard academic bias evaluation paper; the Apple affiliation evaluating non-Apple models is mildly interesting but not inherently controversial."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The WinoIdentity benchmark and code are publicly released at GitHub, allowing practitioners and researchers to reproduce the evaluation immediately."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Apple Research authorship and COLM 2025 venue provide moderate brand recognition; not from a top-tier AI lab primarily known for LLMs."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44582856",
         "title": "A LoD of Gaussians: Ultra-Large Scale Reconstruction with External Memory",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44582856",
         "created_at": "2025-07-16T14:40:05Z"
       },
       {
         "hn_id": "44561157",
         "title": "The Kinematic Age of 3I/Atlas and Its Implications for Early Planet Formation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44561157",
         "created_at": "2025-07-14T15:12:25Z"
       },
       {
         "hn_id": "45466398",
         "title": "Who's Advertising to Your AI?",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=45466398",
         "created_at": "2025-10-03T18:53:26Z"
       },
       {
         "hn_id": "45867965",
         "title": "FPI-Det: A Face–Phone Interaction Dataset for Phone-Use Detection",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45867965",
         "created_at": "2025-11-09T18:43:52Z"
       }
     ],
     "top_points": 2,
     "total_points": 6,
     "total_comments": 1
   }
 }