ai-research-survey

scan.json (25782B)

---

 {
   "paper": {
     "title": "Attention Pruning: Automated Fairness Repair of Language Models via Surrogate Simulated Annealing",
     "authors": [
       "Vishnu Asutosh Dasu",
       "Md Rafi ur Rashid",
       "Vipul Gupta",
       "Saeid Tizpaz-Niari",
       "Gang Tan"
     ],
     "year": 2025,
     "venue": "ICSE '26",
     "arxiv_id": "2503.15815",
     "doi": "10.1145/3744916.3773115"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We open-source our implementations' and provides a Bitbucket link: https://bitbucket.org/psu_soslab/attention_pruning (Section 5, footnote 2)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: HolisticBias and WikiText-2. Both are standard public benchmarks that the authors did not modify, making them accessible to reproducers."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Technical Details) specifies python=3.10, torch==2.2.2, transformers==4.29.0, torch==2.4.1, Ubuntu 20.04.6 LTS and Ubuntu 22.04.3 LTS, Intel Xeon Gold 6336Y CPU, NVIDIA RTX A6000 GPUs, and Intel Core i7-7700 CPU. This is sufficiently detailed."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the code is released and environment details are given, the paper itself does not provide step-by-step reproduction instructions or describe a README with commands. The algorithmic details are described at a high level, but specific reproduction steps (e.g., commands to run, configuration files) are not provided in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports bias values with ± standard deviation (e.g., '0.264 ± 0.01'). The Discussion section reports 95% CIs for Cliff's delta (e.g., '95% CI ≈ [−0.994, −0.847]')."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section reports Cliff's delta and Mann-Whitney U tests with p-values (e.g., p = 2.319 × 10^-9 for GPT-J-6B, p = 6.539 × 10^-9 for Llama-2-7B) to validate the practical significance of AP over FASP."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cliff's delta is reported as an effect size measure (δ = −0.968 for GPT-J-6B, δ = −0.9392 for Llama-2-7B). Relative improvements are also reported (e.g., '40.8% improvement', '65.21% improvement')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 6 LLMs and 25 random seeds for the statistical tests but provides no justification for why these specific numbers were chosen. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports standard deviations across 3 seeds (e.g., '0.264 ± 0.01'). Table 3 reports min, max, median, mean, and standard deviation for perplexity distributions in training datasets."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 compares AP against four baselines: FASP (state-of-the-art), Weight Magnitude pruning, Gradient Magnitude pruning, and Random pruning, plus the unpruned baseline LLM."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "FASP (Zayed et al., 2024) is the state-of-the-art fairness-aware pruning method published the year before. Weight Magnitude (Han et al., 2016) and Gradient Magnitude (Michel et al., 2019) are standard general-purpose pruning baselines. The choice of baselines mirrors the setup of the state-of-the-art method."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 presents ablation studies using only one surrogate DNN in the cost function (only bias DNN vs only perplexity DNN vs combined). RQ3 examines the effect of varying epsilon, eta_bias, and running time on results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both bias (HolisticBias metric) and perplexity (WikiText-2) as primary metrics. MSE is used for surrogate DNN validation. Multiple bias types (gender, race, nationality, sexual orientation, age) are also evaluated."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates bias and utility entirely through automated metrics (toxicity scores from a BERT classifier and perplexity). No human evaluation of the generated text quality or fairness is included. Given the paper makes claims about fairness in language generation, human evaluation of whether the outputs are actually perceived as less biased would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The surrogate DNNs use a 95%-5% train-validation split. Final bias results in Table 6 are reported on '33,789 test set prompts' from HolisticBias. Perplexity is evaluated on the WikiText-2 test dataset. Surrogate training uses validation subsets, while final evaluation uses separate test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 provides per-model breakdowns for all 6 LLMs. Table 7 breaks down results by social bias type (race, nationality, sexual orientation, age). Table 4 provides per-model MSE for the surrogate DNNs."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that for GPT-Neo-125M, AP's bias is worse than FASP (Table 6). The Limitations section discusses sub-optimality due to surrogate DNN errors and SA convergence issues. The anomalous Llama-2-7B behavior in Table 5 is discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "GPT-Neo-125M shows AP performing worse on bias than FASP (0.241 vs 0.221). The paper notes that for Llama-2-7B, using only the bias DNN in the cost function yields higher bias than the combined cost function, which is an unexpected negative result they explicitly discuss."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 40% reduction in gender bias' which is supported by Table 6 (GPT-J-6B: 40.8% relative improvement). The claim that AP 'outperforms state-of-the-art bias mitigation strategies' is supported in 4/6 cases for combined bias+perplexity, which the paper honestly reports."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about pruning attention heads reducing bias (ablation-style claims). The experimental design is adequate: they systematically prune different subsets of attention heads and measure the resulting bias/perplexity changes. The ablation in Table 5 validates the contribution of each surrogate DNN component."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states: 'it is unknown whether our method will generalize to all LLMs and bias metrics.' The title and paper scope focus on attention head pruning specifically. They acknowledge limitation to HolisticBias and WikiText-2 datasets due to computational resources."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Threats to Validity section discusses DNN architecture sensitivity, randomness in SA, and dataset limitations. The Discussion section considers that the relationship between bias and perplexity may be complex (Llama-2-7B anomaly). They discuss surrogate DNN accuracy as an alternative explanation for sub-optimal results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies exact model names: Distilgpt-2, GPT-2, GPT-Neo-125M, GPT-Neo-1.3B, GPT-J-6B, Llama-2-7B. These are specific open-source models with fixed weights (not API-accessed models that change over time), so the exact version is deterministic from the model name. transformers==4.29.0 is specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The prompts come from the publicly available HolisticBias dataset. Table 9 shows example prompts used (e.g., 'I don't like intersex boys.'). The bias evaluation procedure using HolisticBias prompts is well-documented via reference to prior work [14, 72]."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports: epsilon values (0.5, 0.3-0.7 range), n_l and n_u bounds, eta_bias ∈ [0.1, 0.2], eta_ppl = 1, DNN architectures (Table 2), dataset sizes (Table 2), train-validation split (95%-5%), 3 seeds for text generation, 25 seeds for statistical tests, SA time limit of 3 hours, sigma ≤ 10 for clamping, token sampling of 40 tokens."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a traditional ML pipeline (surrogate DNN training + simulated annealing optimization), not an LLM agent system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 documents data preprocessing in detail: perplexity clamping with sigma threshold (Equation 22), bias scaling by maximum, biased subsampling to maintain subgroup distribution, and the rationale for each step."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section (Section 7) contains a dedicated 'Limitations' subsection and a 'Threats to validity' subsection, both providing substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Threats to Validity section discusses specific threats: (1) DNN architecture sensitivity ('The results might vary with different architectures'), (2) outlier behavior for specific models (Llama-2-7B in Table 5), (3) limited computational resources restricting to HolisticBias and WikiText-2, (4) unknown generalizability to all LLMs and bias metrics."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: limited to HolisticBias and WikiText-2 datasets, focused on 6 specific LLMs, primarily studies gender bias with extension to race. The Limitations section states 'it is unknown whether our method will generalize to all LLMs and bias metrics.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While HolisticBias and WikiText-2 are public, the surrogate DNN training datasets (the sampled attention head configurations with their bias/perplexity values) are not released. These are the raw experimental data, and without them, independent verification of the surrogate DNN training is not possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 describes the data collection procedure in detail: random state sampling, pruning, prompting with subsets of HolisticBias, computing bias and perplexity, repeating until time limit. Table 2 reports dataset sizes. Table 3 reports dataset statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard benchmarks (HolisticBias, WikiText-2)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: (1) random state sampling with bounds [n_l, n_u], (2) LLM pruning, (3) bias/perplexity computation on subsets, (4) dataset creation with ≥25,000 samples, (5) preprocessing (clamping, scaling, biased subsampling), (6) surrogate DNN training with 95-5 split, (7) SA search, (8) final evaluation on test data."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This material is based upon work supported by the National Science Foundation under Grant No. CNS-2527657 and CNS-2230061.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Pennsylvania State University (Dasu, Rashid, Gupta, Tan) and University of Illinois Chicago (Tizpaz-Niari). The paper evaluates open-source models, not proprietary models from the authors' affiliated institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the National Science Foundation, a U.S. government funding agency that has no financial interest in whether attention pruning reduces bias in specific LLMs."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capability on a benchmark. It evaluates a post-processing pruning technique's effect on bias and perplexity. The models' training cutoffs are irrelevant because the evaluation measures the effect of pruning, not the models' learned knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as training_cutoff_stated: the paper tests a pruning intervention, not pre-trained model knowledge. Train/test overlap for the LLMs' pre-training data is not relevant to the claims."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning: the paper evaluates a bias mitigation technique, not LLM benchmark performance. Contamination of HolisticBias prompts in training data would not affect the validity of comparing pruned vs unpruned models on the same prompts."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports: one round of inference for Llama-2-7B takes ≈13 minutes on NVIDIA RTX A6000 (Section 4.1). Surrogate DNN inference takes ≈0.34 milliseconds on CPU, enabling ≈2,940 states/second, a 2,260,000× speedup. SA search time limit is 3 hours."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section states: 'the one-time offline data collection of 27,000 samples and training of surrogate DNNs takes ≈1900 GPU hours in total across multiple GPUs for Llama-2-7B.' Hardware details (10 NVIDIA RTX A6000 GPUs, 512GB RAM) are provided."
       }
     }
   },
   "claims": [
     {
       "claim": "Surrogate DNNs accurately predict the bias and perplexity of pruned LLMs with low mean squared error (MSE ≤ 0.01 in most cases).",
       "evidence": "Table 4 shows MSE values: bias DNN MSE ranges from 0.0038 (Distilgpt-2) to 0.0073 (GPT-J-6B); perplexity DNN MSE ranges from 0.0005 (Distilgpt-2) to 0.026 (GPT-Neo-1.3B). Only GPT-Neo-1.3B's perplexity DNN exceeds 0.01.",
       "supported": "strong"
     },
     {
       "claim": "Attention Pruning achieves up to 40% reduction in gender bias compared to baseline LLMs.",
       "evidence": "Table 6: GPT-J-6B baseline bias is 0.446 ± 0.013; AP reduces it to 0.264 ± 0.01, a 40.8% relative improvement. AP achieves bias reduction across all 6 LLMs.",
       "supported": "strong"
     },
     {
       "claim": "AP outperforms the state-of-the-art FASP method by finding states with both lower bias and lower perplexity in 4 out of 6 LLMs.",
       "evidence": "Table 6 shows AP achieves better bias AND perplexity than FASP for GPT-2, GPT-Neo-1.3B, GPT-J-6B, and Llama-2-7B (on bias only). For GPT-Neo-125M, AP has worse bias but better perplexity. For Distilgpt-2, AP has better bias but slightly worse perplexity than baseline.",
       "supported": "strong"
     },
     {
       "claim": "The improvement of AP over FASP is statistically significant with large effect sizes.",
       "evidence": "Discussion section: Cliff's delta = −0.968 (95% CI [−0.994, −0.847]) for GPT-J-6B and δ = −0.9392 (95% CI [−0.984, −0.785]) for Llama-2-7B. Mann-Whitney U test: p = 2.319 × 10^-9 and p = 6.539 × 10^-9 respectively, over 25 random seeds each.",
       "supported": "strong"
     },
     {
       "claim": "Reducing gender bias with AP also reduces other social biases (race, nationality, sexual orientation, age) by up to 65%.",
       "evidence": "Table 7 shows consistent reduction across all 6 LLMs and 4 additional bias types. Highest relative improvement is 65.21% for GPT-J-6B age bias (0.069 → 0.024).",
       "supported": "strong"
     },
     {
       "claim": "Surrogate SA achieves a ~2,260,000× speedup over vanilla SA on LLMs.",
       "evidence": "Section 4.3: One round of inference on Llama-2-7B takes ~13 minutes on GPU. Surrogate DNN inference takes ~0.34 milliseconds on CPU, enabling ~2,940 states/second vs ~5 states/hour for vanilla SA.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Attention Pruning (AP) uses surrogate deep neural networks to approximate the effect of pruning attention heads on LLM bias and perplexity, enabling efficient simulated annealing search over the combinatorial space of attention head configurations. AP achieves up to 40% reduction in gender bias while outperforming the state-of-the-art FASP pruning strategy in 4 out of 6 LLMs by finding configurations with both lower bias and lower perplexity. Reducing gender bias via AP also reduces other social biases (race, nationality, sexual orientation, age) by up to 65%. The surrogate approach provides a ~2.26M× speedup over direct SA search on the LLMs, making combinatorial optimization feasible for billion-parameter models.",
   "red_flags": [
     {
       "flag": "Fairness metric relies on automated toxicity classifier",
       "detail": "The bias metric is computed using a BERT-based toxicity classifier trained on the Jigsaw toxic comment dataset. This classifier itself may have biases, and the paper does not validate whether the toxicity scores accurately reflect human perceptions of bias. No human evaluation of the debiased outputs is performed."
     },
     {
       "flag": "Statistical tests only on 2 of 6 models",
       "detail": "Cliff's delta and Mann-Whitney U tests are reported only for GPT-J-6B and Llama-2-7B. The other 4 models, including GPT-Neo-125M where AP underperforms FASP on bias, do not have statistical significance tests."
     },
     {
       "flag": "Surrogate DNN training cost may limit practical applicability",
       "detail": "The one-time offline data collection and training takes ~1900 GPU hours for Llama-2-7B. This significant upfront cost is acknowledged but may limit the practical applicability of the approach to larger or different models."
     }
   ],
   "cited_papers": [
     {
       "title": "Fairness-aware structured pruning in transformers",
       "authors": ["Abdelrahman Zayed", "Gonçalo Mordido", "Samira Shabanian", "Ioana Baldini", "Sarath Chandar"],
       "year": 2024,
       "relevance": "State-of-the-art fairness-aware attention head pruning method that AP is compared against; directly relevant to LLM bias mitigation methodology."
     },
     {
       "title": "NeuFair: Neural Network Fairness Repair with Dropout",
       "authors": ["Vishnu Asutosh Dasu", "Ashish Kumar", "Saeid Tizpaz-Niari", "Gang Tan"],
       "year": 2024,
       "doi": "10.1145/3650212.3680380",
       "relevance": "Prior work on search-based fairness repair using simulated annealing for smaller DNNs; directly informs the AP approach."
     },
     {
       "title": "Information-Theoretic Testing and Debugging of Fairness Defects in Deep Neural Networks",
       "authors": ["Verya Monjezi", "Ashutosh Trivedi", "Gang Tan", "Saeid Tizpaz-Niari"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00136",
       "relevance": "Fairness testing and debugging technique for DNNs using information theory; relevant to the SE fairness testing literature."
     },
     {
       "title": "Bias and fairness in large language models: A survey",
       "authors": ["Isabel O Gallegos", "Ryan A Rossi", "Joe Barrow"],
       "year": 2024,
       "relevance": "Comprehensive survey of bias and fairness in LLMs; provides context for the fairness mitigation landscape."
     },
     {
       "title": "\"I'm sorry to hear that\": Finding New Biases in Language Models with a Holistic Descriptor Dataset",
       "authors": ["Eric Michael Smith", "Melissa Hall", "Melanie Kambadur", "Eleonora Presani", "Adina Williams"],
       "year": 2022,
       "arxiv_id": "2205.09209",
       "relevance": "HolisticBias dataset and metric used as the primary fairness evaluation benchmark in this paper."
     },
     {
       "title": "BOLD: Dataset and Metrics for Measuring Biases in Open-Ended Language Generation",
       "authors": ["Jwala Dhamala", "Tony Sun", "Varun Kumar"],
       "year": 2021,
       "doi": "10.1145/3442188.3445924",
       "relevance": "Foundational bias evaluation dataset and methodology that HolisticBias builds upon."
     },
     {
       "title": "Are Sixteen Heads Really Better than One?",
       "authors": ["Paul Michel", "Omer Levy", "Graham Neubig"],
       "year": 2019,
       "arxiv_id": "1905.10650",
       "relevance": "Influential work on attention head pruning based on gradient magnitude; used as a baseline in this paper."
     },
     {
       "title": "Black Box Fairness Testing of Machine Learning Models",
       "authors": ["Aniya Aggarwal", "Pranay Lohia", "Seema Nagar", "Kuntal Dey", "Diptikalyan Saha"],
       "year": 2019,
       "doi": "10.1145/3338906.3338937",
       "relevance": "Early work on fairness testing in SE community; relevant to the systematic evaluation of ML model fairness."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2024,
       "arxiv_id": "2303.08774",
       "relevance": "Key reference for LLM capabilities; motivates the need for fairness repair in advanced language models."
     },
     {
       "title": "A genetic programming approach to automated software repair",
       "authors": ["Stephanie Forrest", "ThanhVu Nguyen", "Westley Weimer", "Claire Le Goues"],
       "year": 2009,
       "doi": "10.1145/1569901.1570031",
       "relevance": "Foundational work on search-based software repair that inspires the simulated annealing approach used in AP."
     },
     {
       "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "One of the largest LLMs evaluated in the paper; relevant to understanding LLM capabilities and bias."
     }
   ]
 }