ai-research-survey

scan-v5.json (25686B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Does Reasoning Introduce Bias? A Study of Social Bias Evaluation and Mitigation in LLM Reasoning",
     "authors": [
       "Xuyang Wu",
       "Jinming Nian",
       "Ting-Ruen Wei",
       "Zhiqiang Tao",
       "Hsin-Tai Wu"
     ],
     "year": 2025,
     "venue": "Conference on Empirical Methods in Natural Language Processing",
     "arxiv_id": "2502.15361",
     "doi": "10.18653/v1/2025.findings-emnlp.1006"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All four major abstract claims — first systematic evaluation of reasoning bias, accuracy-without-bias-mitigation finding, bias amplification in ambiguous contexts, and ADBP outperforming SfRP — are supported by Tables 1–2 and Figures 3–6.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The SfRP ablation (remove biased steps → re-run instruction model) constitutes an intervention test for the causal claim that biased reasoning causes errors; robustness of the LLM judge across prompt variants (Section 5.1, Table 5) partially validates the oracle, though human annotation is absent.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Conclusions like 'reasoning-based models do not mitigate biases' are stated broadly across the paper while evidence covers only five open-source models plus cited OpenAI system card numbers on a single English-language benchmark (BBQ); computational constraints also excluded the full DeepSeek-R1 model.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly notes that incorrect answers can arise without biased reasoning (white lines in Figure 3b) and identifies non-negative polarity misinterpretation as an independent source of errors (Figure 4b, Appendix A.7).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses distinct metrics for different constructs: exact-match accuracy for prediction correctness and formally-defined bias scores (Equations 2–3) for stereotype expression, and keeps these separate throughout analysis.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present, separate from the conclusion, addressing the LLM judge, mitigation design, computational constraints, and refusal behavior.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: LLM-as-a-judge uncertainty without human annotation verification, inability to test full DeepSeek-R1 due to compute costs, distilled models potentially carrying inherent biases, and rare refusal behavior due to BBQ's controlled framing.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope to distilled DeepSeek-R1 variants rather than the full model, and to the BBQ benchmark in zero-shot English-language settings; computational and cost limitations are explicitly named.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment or funding section appears anywhere in the paper; funding sources are entirely absent.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed on the title page (Santa Clara University, Rochester Institute of Technology, Docomo Innovations).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding source is disclosed, making this criterion not applicable.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines social bias (stereotype-based associations), SfRP, ADBP, ambiguous vs. disambiguated contexts, and provides formal equations for both accuracy and bias score metrics.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are enumerated: first evaluation of bias in reasoning steps (not just outputs), empirical finding that reasoning amplifies stereotypes, and the ADBP mitigation strategy.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The related work section distinguishes this study from prior bias work (which focused on outputs, not reasoning steps) and from prior CoT studies (which focused on math/code), clearly positioning the contribution as a gap-filling study.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The abstract links directly to https://github.com/elviswxy/LLM_reasoning_bias for evaluation and mitigation code.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "BBQ is a publicly available standard benchmark (Parrish et al., 2022); no novel dataset was created.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Only hardware (NVIDIA A100 GPUs) is mentioned; no requirements.txt, Dockerfile, or Python/library versions are provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": true,
           "justification": "Algorithm 1 specifies the ADBP procedure step-by-step, all prompts are provided in Appendix A.2, model HuggingFace links are given, and code is released.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are reported for the main accuracy or bias score results in Tables 1–2.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Comparative claims (e.g., 'ADBP outperforms SfRP in most cases') are made without any statistical significance tests.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Numerical accuracy improvements from mitigation are reported in absolute terms (e.g., +0.517 in Case 1, +0.717 in Case 3), providing quantified effect sizes.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The full BBQ dataset (58,492 examples) is used without any power analysis or justification for whether this is sufficient to detect the observed effect sizes reliably.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "The LLM-as-a-judge scores each step 5 times and takes majority vote, but variance across those runs is not reported; main accuracy/bias results show no spread.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "ADBP is compared against SfRP-based mitigation, Self-debiasing via Explanation (Gallegos et al., 2025), and Combined Debiasing Prompt (Liu et al., 2025a) in Figure 6 and Table 2.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All mitigation baselines (Gallegos et al., 2025; Liu et al., 2025a) are from 2025, published at NAACL 2025 and contemporaneous with this work.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "The SfRP component is an ablation: removing biased steps from DeepSeek reasoning and re-querying the base model isolates the contribution of biased reasoning to incorrect predictions (Figure 5).",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Two distinct metrics are used throughout: accuracy (Acc) and bias score (Bias), each split by context type (ambiguous/disambiguated).",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "The paper explicitly states 'We did not conduct human labeling to verify [LLM-as-a-judge] reliability due to the extremely high cost of manual annotation.'",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "This is an inference-only evaluation study; no model training or train/test splits are involved.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Tables 1a and 1b provide per-category accuracy and bias scores across all 11 BBQ categories for each evaluated model.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.6 provides full qualitative failure examples (biased reasoning leading to wrong answers across Religion, Age, Disability categories), and Section 4.3 analyzes failure patterns systematically.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports that ADBP underperforms SfRP for the Qwen-32B base model (Case 3, Table 2) and that reasoning models fail to reduce bias despite improving accuracy.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact HuggingFace model IDs are provided via footnotes for all open-source models (e.g., deepseek-ai/DeepSeek-R1-Distill-Llama-8B); OpenAI results are sourced from the o3-mini system card.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "All prompts are provided verbatim in Appendix A.2 (Figures 7–12), including evaluation prompts for instruction-tuned models, DeepSeek models, LLM-as-a-judge, and ADBP.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "The paper states generation parameters follow each model's system card without specifying temperature, top-p, or other decoding hyperparameters.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; this is a direct inference evaluation study.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Reasoning step extraction (newline splitting of <think> tokens), k=100 bin normalization for visualization, exact-match response normalization with regex, and LLM-as-a-judge 5-run majority voting are all documented.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "While code is released, the intermediate scored outputs (per-step LLM judge bias scores for all reasoning traces) are not explicitly released as a dataset artifact.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The BBQ dataset is described (Table 3, dataset statistics), model inference procedure is described, and the LLM-as-a-judge scoring procedure (GPT-4o, 5 runs, majority vote) is detailed in Section 3.4.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; the study uses a pre-existing benchmark dataset.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from BBQ input → model inference → reasoning step extraction → LLM-as-a-judge scoring → accuracy/bias computation is described across Sections 3–4.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff dates are stated for any of the evaluated models (DeepSeek, Llama, Qwen, OpenAI).",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "BBQ (published 2022) could be in any of the 2024–2025 model training corpora; the paper does not discuss or test for this possibility.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "BBQ predates all evaluated models' training, making contamination plausible, but the paper does not address whether BBQ examples were in the models' pretraining data.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions 'monetary cost' as a drawback of the LLM-as-a-judge approach (Section 6) but provides no actual cost figures for GPT-4o calls or model inference.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "NVIDIA A100 GPUs are mentioned but no total compute budget (GPU-hours, cost) is stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Reasoning-based models (DeepSeek-R1 variants) improve prediction accuracy compared to base instruction-tuned models but do not reduce social bias — in many categories they exhibit equal or worse bias scores.",
       "evidence": "Table 1: DeepSeek-8B achieves highest accuracy in all 11 categories but shows similar or worse bias in 9/11 ambiguous categories vs. Llama-8B.",
       "supported": "strong"
     },
     {
       "claim": "Biased reasoning steps are significantly more prevalent in incorrect predictions than in correct predictions across multiple BBQ categories.",
       "evidence": "Figure 3 and Table 6 show higher average bias scores in subsets where the reasoning model is wrong (e.g., Age: 1.06 vs. 0.23; SES: 1.57 vs. 0.46; Religion: 1.55 vs. 0.64).",
       "supported": "strong"
     },
     {
       "claim": "Removing biased reasoning steps (SfRP) and re-querying instruction-tuned models consistently improves accuracy on previously failed cases.",
       "evidence": "Figure 5: SfRP improves accuracy by +0.517 (Case 1) and +0.717 (Case 3); even in both-fail cases accuracy increases by +0.100 and +0.526.",
       "supported": "strong"
     },
     {
       "claim": "ADBP outperforms SfRP mitigation in most cases by using answer distribution shifts across incremental reasoning steps as a proxy for bias.",
       "evidence": "Table 2: ADBP exceeds SfRP accuracy in Cases 1, 2, and 4 for both model families; ADBP corrects 38–60% of initially incorrect cases vs. SfRP's 24–44%. Exception is Qwen-32B Case 3.",
       "supported": "moderate"
     },
     {
       "claim": "Ambiguity amplifies bias in reasoning models: reasoning-based models that outperform base models in disambiguated contexts often fail to do so in ambiguous contexts.",
       "evidence": "Section 4.2: OpenAI o1, o1-mini, o3-mini underperform GPT-4o in ambiguous contexts; DeepSeek-32B fails to consistently outperform Qwen-32B under ambiguity in categories like Age and Physical Appearance.",
       "supported": "moderate"
     },
     {
       "claim": "Bias tends to intensify in later reasoning steps: once a biased step appears, the model tends to persist along a faulty trajectory.",
       "evidence": "Figure 3b and 3d show bias accumulating toward the end of reasoning chains for incorrect predictions, while correct predictions show isolated, non-propagating bias.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational",
     "case-study"
   ],
   "key_findings": "Reasoning-based LLMs (DeepSeek-R1 variants, Marco-o1) improve accuracy on the BBQ social bias benchmark over base instruction-tuned models but do not mitigate social bias — in ambiguous contexts they often amplify stereotypes, particularly for categories like Age, Physical Appearance, and SES. Biased reasoning steps are strongly correlated with incorrect predictions, and removing them via SfRP consistently improves accuracy, supporting a causal role for biased reasoning in prediction errors. The proposed ADBP method, which uses answer distribution shifts across incremental reasoning steps as a bias proxy, outperforms SfRP and prompt-only debiasing baselines in most tested scenarios without requiring an external judge at inference time.",
   "red_flags": [
     {
       "flag": "LLM judge unvalidated",
       "detail": "The paper explicitly acknowledges it did not conduct human annotation to validate the LLM-as-a-judge reliability; this oracle underpins the primary causal claim (bias → incorrect predictions) and the SfRP intervention."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Comparative claims (ADBP outperforms SfRP, reasoning models amplify bias vs. base models) are made without any significance tests despite per-category sample sizes that would permit them."
     },
     {
       "flag": "BBQ contamination unaddressed",
       "detail": "BBQ was published in 2022; all evaluated models (DeepSeek-R1, Llama-3.1, Qwen2.5) were trained after 2022. Potential benchmark contamination in pretraining data is not discussed."
     },
     {
       "flag": "Broad generalization from few models",
       "detail": "Conclusions about 'reasoning-based LLMs' in general are drawn from 2–3 open-source model families on a single English-language benchmark, without the full DeepSeek-R1 or diverse proprietary models."
     },
     {
       "flag": "Depth analysis limited to 3/11 categories",
       "detail": "The reasoning step bias analysis (Figure 3, Section 5.2) is demonstrated for only Age, Religion, and SES categories out of 11 BBQ categories, limiting generalizability of the mechanism claim."
     }
   ],
   "cited_papers": [
     {
       "title": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
       "relevance": "Primary evaluation benchmark; defines ambiguous/disambiguated context structure and bias scoring methodology used throughout."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "relevance": "Foundational CoT paper that motivates evaluating whether reasoning chains introduce bias."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "relevance": "Primary evaluated models (distilled variants DeepSeek-8B and DeepSeek-32B) are sourced from this work."
     },
     {
       "title": "Decoding Biases: Automated Methods and LLM Judges for Gender Bias Detection in Language Models",
       "relevance": "Provides the LLM-as-a-judge methodology adopted for per-step bias scoring."
     },
     {
       "title": "On Second Thought, Let's Not Think Step by Step! Bias and Toxicity in Zero-Shot Reasoning",
       "relevance": "Most closely related prior work showing CoT prompting can increase harmful outputs in sensitive domains."
     },
     {
       "title": "Evaluating Gender Bias in Large Language Models via Chain-of-Thought Prompting",
       "relevance": "Related prior work on CoT and gender bias; key differentiator is that this paper analyzes native reasoning chains rather than prompted CoT."
     },
     {
       "title": "Self-Debiasing Large Language Models: Zero-Shot Recognition and Reduction of Stereotypes",
       "relevance": "One of the two mitigation baselines compared against ADBP in Figure 6."
     },
     {
       "title": "Evaluating and Mitigating Social Bias for Large Language Models in Open-Ended Settings",
       "relevance": "Combined Debiasing Prompt baseline compared against ADBP in Figure 6."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "ADBP is a lightweight, annotation-free method applicable at inference time to any model that exposes reasoning traces, making it directly usable by practitioners deploying reasoning LLMs."
     },
     "surprise_contrarian": {
       "score": 3,
       "justification": "The central finding — that reasoning capability improves accuracy but simultaneously amplifies social bias — directly contradicts the intuition that 'better reasoning = better alignment.'"
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that widely deployed reasoning models (DeepSeek-R1, o1) systematically express stereotypes against protected groups in QA tasks, raising concrete deployment safety concerns."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Challenges DeepSeek-R1 and OpenAI o1's implicit safety claims but without naming adversarial parties; low drama framing."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is released and models (DeepSeek-R1 distilled variants) are freely available on HuggingFace, enabling reproduction; BBQ dataset is public."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "DeepSeek-R1 and OpenAI o1/o3-mini are high-profile models; EMNLP venue adds credibility, though the institution (Santa Clara University) is not a top-tier AI lab."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "43405094",
         "title": "Politicians' misinformation behavior and public engagement, in 4 countries",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43405094",
         "created_at": "2025-03-18T21:03:45Z"
       }
     ],
     "top_points": 3,
     "total_points": 3,
     "total_comments": 0
   }
 }