ai-research-survey

scan.json (24831B)

---

 {
   "paper": {
     "title": "Bias Unveiled: Investigating Social Bias in LLM-Generated Code",
     "authors": ["Lin Ling", "Fazle Rabbi", "Song Wang", "Jinqiu Yang"],
     "year": 2025,
     "venue": "AAAI 2025",
     "arxiv_id": "2411.10351",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository linked in the paper: https://github.com/janeeyre912/fairness_testing_code_generation. Code and datasets URLs are provided at the top of the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset (SocialBias-Bench) is released at https://github.com/janeeyre912/fairness_testing_code_generation/tree/master/dataset, linked directly in the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is mentioned in the paper. The paper mentions using textX DSL framework and specific LLM APIs but does not specify library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper links to code and data but does not include a README with commands or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports CBS and BLS as point estimates (e.g., '60.58%', '28.34%') without confidence intervals or error bars. Figure 4 shows temperature effects but without uncertainty bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses t-tests to assess statistical significance of bias mitigation strategies. Table 5 marks statistically significant changes with asterisks (*), and the paper states 'If p < 0.05, the difference is statistically significant.' Temperature effects are also evaluated with p-values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'the overall bias decreased to 29.15% from 60.58%' after the first iteration, and 'reduce social bias in LLM-generated code by up to 90%'. CBS values are provided for each demographic before and after mitigation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 343 tasks and 1,715 code snippets per model (5 per task), but there is no justification for why 343 tasks or 5 snippets per task were chosen. No power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for the CBS or BLS metrics across the 5 code snippets generated per task. Only aggregated percentages are presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against two prior works (Liu et al. 2023 and Huang et al. 2023) that also investigate social bias in code generation. The default (no mitigation) condition serves as the baseline for mitigation experiments."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared prior works (Liu et al. 2023 and Huang et al. 2023) are contemporary, both published in 2023, which is recent relative to the 2024/2025 submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation studies on temperature effects (Section 'Effects of temperature' with Figure 4) and prompt variations (judgemental words). Three mitigation strategies are tested incrementally (COT, positive role-play + COT, iterative prompting)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: Code Bias Score (CBS) for overall bias severity, Bias Leaning Score (BLS) for fine-grained bias direction, and Pass@attribute for functional correctness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes claims about real-world bias implications and fairness but relies entirely on automated metamorphic testing. No human evaluation of the generated code's fairness or quality is included. Given the claims about real-world social impact, human judgment on whether detected biases are meaningful would be relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a supervised learning task with train/test splits. The framework generates test cases per task definition and evaluates all code snippets. No model training or tuning on the dataset occurs."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by demographic dimension (age, gender, religion, race, employment status, marital status, education) in Tables 3 and 4, and by task category in Table 2. Radar charts (Figure 3) show per-demographic patterns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where mitigation strategies failed or worsened bias: 'COT prompt increases CBSdemographic for all dimensions and the overall CBS' and 'role-playing can sometimes reinforce biases when sensitive attributes are unintentionally embedded in the context or reasoning steps.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that COT prompting and positive role-play + COT prompting were not effective and sometimes increased bias. For GPT-3.5, COT increased CBS from 60.58% to 72.65%. This is a clear negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims: (1) severe bias in all four LLMs — supported by Table 3 showing CBS 28-60%. (2) Dialogue with Solar reduces bias by up to 90% — supported by Table 5 showing iterative prompting reducing CBS from 60.58% to 8.77% for GPT-3.5. (3) Code and data publicly available — GitHub links provided."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about mitigation strategies reducing bias ('iterative prompting effectively reduces bias'). The study design uses controlled before/after comparisons with the same tasks, models, and dataset, which is adequate for these claims. Statistical tests support significance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Investigating Social Bias in LLM-Generated Code' is broad but results are limited to 4 specific LLMs, 343 human-centered tasks in 7 categories, and Python code generation. The paper does not explicitly bound the generalization — it does not state these results may not hold for other LLMs, programming languages, or non-human-centered tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed biases. For instance, it does not consider whether the bias results could be influenced by the specific prompt format used, the DSL-generated test case design, or whether the task definitions themselves may embed implicit assumptions. No threats-to-validity section or alternative explanations are provided."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions are specified: GPT-3.5-turbo-0125, codechat-bison@002, CodeLlama-70b-instruct-hf, and claude-3-haiku-20240307. These include version identifiers and snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides the actual prompt structure in Figure 1 (sub-figure b) showing the code prompt format, and the exact text of mitigation prompts: 'Let's think step by step. Consider the condition carefully to ensure fairness' for COT and 'You are a fair and impartial code developer...' for role-play. The DSL-generated prompts follow a documented template with actual examples."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings are explicitly discussed and varied (0.2 to 1.0) in the temperature ablation study (Figure 4). The default temperature is implied as 1.0 (the default setting). The paper also generates 5 code snippets per task."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The Solar framework's workflow is described in detail: DSL-based prompt generation, test case generation via textX, metamorphic testing with attribute mutation, iterative feedback loop. Figure 1 shows the full pipeline, and the methodology section describes each component."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the task generation process: manually crafted 2-3 tasks per category, GPT-4o generated 60 scenarios per category, then 'remove the duplicate and unrelated generated tasks and then adjust some related attributes.' Second author cross-checked. The final count of 343 tasks is reported."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section is present in the paper. The conclusion briefly mentions future work ('expand the datasets to include more scenarios') but this is not a substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues such as the representativeness of the 343 tasks, the appropriateness of the bias definition used, or whether the DSL-generated test cases adequately capture real-world bias scenarios."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There are no explicit statements about which populations, settings, or languages are excluded from the claims. The preliminary section limits bias to 'those against different demographics in human-centered tasks' but does not discuss what this excludes."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset and code are released on GitHub (https://github.com/janeeyre912/fairness_testing_code_generation). The task definitions and generated code snippets should be verifiable through the released artifacts."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: 2-3 manually crafted tasks per category, GPT-4o used to generate additional scenarios, filtering for duplicates and unrelated tasks, and cross-checking by the second author. Categories and task counts are listed in Table 2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The data is a benchmark dataset of coding tasks, not data from human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from task definition to code prompt generation to test case generation to bias evaluation is documented in the Methodology section with Figure 1 showing the workflow. The filtering step for the dataset is described (remove duplicates and unrelated tasks, cross-check by second author)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Concordia University and York University. These are academic institutions, not companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper does not state whether it is unfunded or simply omits funding information."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLMs on code generation tasks but does not state the training data cutoff dates for any of the four models tested. This is relevant because the task structures could potentially appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the task definitions or similar prompts could have appeared in the training data of the subject LLMs. The SocialBias-Bench dataset was partially generated by GPT-4o, raising questions about potential overlap with GPT-3.5 training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not address contamination risk. While SocialBias-Bench is newly created, the task structures are based on common social scenarios that could appear in training data. Additionally, the paper references HumanEval performance for model comparison but does not discuss contamination for that benchmark either."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated using the Solar framework on LLM-generated code."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM-generated code for bias using automated testing."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The demographics in the paper refer to the demographic dimensions used in the bias evaluation framework, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental assignment of participants to conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no blinding applicable to automated LLM evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time are reported. The framework calls LLMs multiple times per task (5 code snippets per task, plus iterative prompting with up to 3 iterations) but no cost information is provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The total API spend for 4 models x 1715 snippets plus mitigation experiments is not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "All four LLMs exhibit severe social bias in code generation, with overall CBS ranging from 28.34% to 60.58%.",
       "evidence": "Table 3 shows CBS_overall: GPT-3.5-turbo-0125 at 60.58%, codechat-bison@002 at 40.06%, CodeLlama-70b-instruct-hf at 28.34%, and claude-3-haiku-20240307 at 36.33%.",
       "supported": "strong"
     },
     {
       "claim": "Iterative prompting with Solar feedback can reduce social bias in LLM-generated code by up to 90%.",
       "evidence": "Table 5 shows GPT-3.5-turbo-0125 CBS dropping from 60.58% to 8.77% after 3 iterations (85.5% reduction). The abstract claims 'up to 90%' but the data shows approximately 85.5% for the model presented in detail. Other models' detailed results are in the artifact.",
       "supported": "moderate"
     },
     {
       "claim": "Chain-of-Thought prompting and positive role-playing are not effective for bias mitigation and can increase bias.",
       "evidence": "Table 5 shows GPT-3.5 CBS increased from 60.58% to 72.65% with COT and to 68.66% with P-COT. The paper states 'all the subject LLMs do not exhibit a significant change in the CBS_overall' for these strategies.",
       "supported": "strong"
     },
     {
       "claim": "Bias is most severe in age, gender, and employment status demographics across all tested LLMs.",
       "evidence": "Table 3 shows consistently higher CBS values for these three demographics across all four models. For GPT-3.5: age 31.25%, gender 20.93%, employment 33.24%, while religion (16.44%) and race (19.42%) are lower.",
       "supported": "strong"
     },
     {
       "claim": "Temperature settings significantly affect bias in code generation.",
       "evidence": "Figure 4 and the temperature ablation section show CodeLlama CBS rising from 28.34% to 65.19% as temperature decreases from 1.0 to 0.2. Other models also show significant changes at specific temperatures.",
       "supported": "moderate"
     },
     {
       "claim": "Iterative prompting improves functional correctness while reducing bias.",
       "evidence": "Table 5 shows Pass@attribute increasing from 66.60% (default) to 85.66% after 3 iterations for GPT-3.5-turbo-0125.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes Solar, a fairness framework for evaluating social bias in LLM-generated code using metamorphic testing. Evaluation on four LLMs (GPT-3.5-turbo, codechat-bison, CodeLlama-70b, claude-3-haiku) reveals severe bias across all models, with CBS ranging from 28-61%. Age, gender, and employment status show the highest bias. Iterative prompting using Solar's feedback effectively reduces bias by up to ~85%, while Chain-of-Thought and role-playing prompting strategies are ineffective or counterproductive.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations section or threats-to-validity discussion, which is unusual for a venue like AAAI. There is no discussion of potential weaknesses in the task design, bias definition, or generalizability."
     },
     {
       "flag": "Dataset partially generated by GPT-4o without contamination analysis",
       "detail": "The SocialBias-Bench dataset was partially generated using GPT-4o, but one of the test subjects is GPT-3.5-turbo, which shares training data infrastructure. No analysis of potential contamination or circularity is provided."
     },
     {
       "flag": "Selective presentation of mitigation results",
       "detail": "Full mitigation results are shown only for GPT-3.5-turbo in Table 5 'due to space limits.' The claim of 'up to 90% bias reduction' appears based on the best-case result, and the reader cannot verify this for all models from the paper alone."
     },
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "Despite generating 5 code snippets per task, no variance or standard deviation across snippets is reported. The stability of the bias measurements is unknown."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "The paper tests 4 specific LLMs on Python code generation for human-centered tasks but draws broad conclusions about 'social bias in LLM-generated code' without bounding the scope of the findings."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational HumanEval benchmark for evaluating LLM code generation, used in this paper for model performance comparison."
     },
     {
       "title": "Uncovering and quantifying social biases in code generation",
       "authors": ["Yujia Liu", "Xiaobao Chen"],
       "year": 2023,
       "relevance": "Prior work on social bias in LLM code generation at NeurIPS, which this paper extends with more comprehensive evaluation and mitigation strategies."
     },
     {
       "title": "Bias Testing and Mitigation in LLM-based Code Generation",
       "authors": ["Dong Huang", "Qingwen Bu"],
       "year": 2023,
       "relevance": "Concurrent work on bias testing in text-to-code LLM tasks, directly compared with Solar framework in this paper."
     },
     {
       "title": "Bias and fairness in large language models: A survey",
       "authors": ["Isabel O. Gallegos"],
       "year": 2023,
       "arxiv_id": "2309.00770",
       "relevance": "Comprehensive survey of bias and fairness in LLMs that motivates the study of bias in code generation specifically."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Major open-source code generation model relevant to evaluating LLM code generation capabilities."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the four LLMs evaluated in this paper for code generation bias."
     },
     {
       "title": "FairBench: A Four-Stage Automatic Framework for Detecting Stereotypes and Biases in Large Language Models",
       "authors": ["Yanhong Bai"],
       "year": 2023,
       "arxiv_id": "2308.10397",
       "relevance": "Framework for detecting bias in LLMs, related to the fairness evaluation methodology used in this paper."
     },
     {
       "title": "Fairness Improvement with Multiple Protected Attributes: How Far Are We?",
       "authors": ["Zhenpeng Chen", "Jie M. Zhang", "Federica Sarro", "Mark Harman"],
       "year": 2024,
       "relevance": "ICSE paper on fairness with multiple protected attributes, closely related to the multi-demographic bias evaluation in this paper."
     },
     {
       "title": "An empirical survey of the effectiveness of debiasing techniques for pre-trained language models",
       "authors": ["Nicholas Meade", "Elinor Poole-Dayan", "Siva Reddy"],
       "year": 2021,
       "arxiv_id": "2110.08527",
       "relevance": "Survey of debiasing techniques for language models, relevant context for bias mitigation strategies."
     },
     {
       "title": "Biasasker: Measuring the bias in conversational AI system",
       "authors": ["Yuxiang Wan"],
       "year": 2023,
       "relevance": "FSE paper on measuring bias in conversational AI, related methodology for automated bias detection."
     }
   ]
 }