ai-research-survey

scan.json (22705B)

---

 {
   "paper": {
     "title": "Auditing Fairness under Model Updates: Fundamental Complexity and Property-Preserving Updates",
     "authors": ["Ayoub Ajarra", "Debabrota Basu"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.05909",
     "doi": "10.48550/arXiv.2601.05909"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a link to a supplementary code repository: https://anonymous.4open.science/r/Auditors-with-prospects-050F (Appendix H). This is an anonymous link, indicating review stage, but a working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The experiments use two publicly available datasets: COMPAS (Angwin et al. 2016) and Student Performance (Cortez and Silva 2008). These are standard public benchmarks the authors did not modify."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "While the paper specifies the hardware ('11th Gen Intel Core i7-1185G7 processor (3.00 GHz, 8 cores) with 32.0 GiB of RAM') and mentions scikit-learn, there is no requirements.txt, Dockerfile, or detailed library version listing beyond mentioning scikit-learn's default solver."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Implementation details and instructions for reproducing our results are provided in the supplementary code repository' but does not include step-by-step reproduction instructions in the paper itself. The anonymous repository link may contain them, but the paper text does not."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The figures (Figures 4 and 6) show shaded bands around estimation curves, representing error/uncertainty ranges across sample budgets. Concentration bounds (Theorem 8) provide theoretical confidence guarantees."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about convergence and accuracy of their estimator but does not use formal statistical significance tests (no p-values, t-tests, etc.). Comparisons are shown graphically without formal testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports specific estimation errors (e.g., 2.33×10^-2 for COMPAS/MLP) and ratio errors (e.g., 2.5×10^-3) at a budget of 1000 samples, providing magnitude context for results."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The theoretical analysis (Theorems 2, 4, 7, 8) provides formal sample complexity bounds that justify the required number of samples. The experiments vary budget from 200 to 1000 to show convergence behavior."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figures 4 and 6 show shaded bands around the main estimation curves across different sample budgets, indicating spread/variance in the estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares its framework conceptually to Yan and Zhang (2022) but does not include empirical comparisons against any baseline methods. The experiments only evaluate the authors' own EPO algorithm."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No empirical baselines are included. The paper discusses Yan and Zhang (2022) theoretically but does not run their method for comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is presented. The paper could have ablated components such as the prospect class construction vs. simple SP estimation, but does not."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates using multiple metrics: estimation error of statistical parity, prospect ratio error, and runtime (Table 1, Figures 4, 6)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper with synthetic/benchmark experiments on fairness auditing. Human evaluation of outputs is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe a train/test split in its experimental evaluation. Results are shown across varying labeling budgets from the same datasets without explicit held-out test set separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (COMPAS vs. Student Performance) and per strategic class (MLP vs. Random Forest), shown in Table 1 and Figure 4/6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Proposition 9 establishes that model classes with infinite VC dimension are not weakly or strongly auditable, identifying a clear failure boundary. Section 6 discusses limitations of the approach."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports impossibility results: Proposition 9 proves infinite VC classes are not auditable, and Appendix E.2 shows weak auditability does not extend to all infinite VC classes. These are genuine negative/impossibility results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (i) characterizing information complexity of allowable updates via SP dimension, (ii) distribution-free auditing bounds, and (iii) extension to other objectives are all supported by the theorems (Theorems 2, 4, 7, 8) and Appendix B."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper establishing complexity bounds and proposing algorithms. It does not make causal claims about interventions or outcomes."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about scope: theoretical results specify exactly which hypothesis classes (finite, infinite VC, infinite VC dimension) each result applies to. Experimental claims are bounded to two specific datasets and two strategic model classes."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the experimental results. It does not consider whether other factors could explain the observed convergence patterns or prospect ratio estimates beyond its theoretical framework."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'logistic regression classifier with l2 regularization, trained on the original labels using scikit-learn's default solver' and 'multi-layer perceptron (MLP)' and 'random forest' but does not specify scikit-learn version, MLP architecture details (layers, activation functions, hyperparameters), or random forest configuration."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use LLM prompting. It is a theoretical/experimental paper on fairness auditing using traditional ML models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'scikit-learn's default solver' for logistic regression and tolerance threshold epsilon=0.005 but does not report MLP or random forest hyperparameters (e.g., number of layers, hidden units, number of trees, max depth). The number of sampled hypotheses for the finite approximation is also not specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a traditional ML/theoretical paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states it uses COMPAS with Caucasian/non-Caucasian groups and Student Performance with Female/Male groups but does not describe any preprocessing steps, feature selection, or how the datasets were prepared for the experiments."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Discussion and Future Work' discusses limitations, including the restriction to traditional predictive models and the need for extensions to interactive settings, property-preserving architectures, and manipulation-proof definitions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The discussion section identifies future directions but does not discuss specific threats to validity of the experimental results, such as the small scale of experiments (only 2 datasets), the use of synthetic strategic classes, or the gap between theoretical assumptions and practical settings."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states scope boundaries: 'While our focus was on traditional predictive models' and identifies specific extensions needed (interactive settings, LLMs, manipulation-proof definitions). Proposition 9 clearly delimits what classes cannot be audited."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (COMPAS, Student Performance) that anyone can independently access and verify."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper simply references the COMPAS dataset (Angwin et al. 2016) and Student Performance dataset (Cortez and Silva 2008) without describing how they were obtained or any steps taken for this study."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe the pipeline from raw datasets to the experimental setup. It is unclear how features were selected, how the logistic regression 'true model' was trained, how the strategic class models were sampled, or how many models were included in the finite approximation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper. The authors are affiliated with Inria/CNRS/Univ. Lille but no grants or funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: 'Équipe Scool, Univ. Lille, Inria, CNRS, Centrale Lille, UMR 9189- CRIStAL, France'."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be assessed. The absence of funding disclosure is itself a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It trains traditional ML models (logistic regression, MLP, random forest) on fairness datasets to test an auditing framework."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model benchmark evaluation is involved. The paper tests an auditing algorithm, not model knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model benchmark evaluation is involved. Contamination is not relevant to this auditing framework study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports runtime in milliseconds per sample (1.2-1.9 ms). Figure 6 shows runtime scaling across budgets. The paper states the EPO oracle requires ~3 ms per sample for random forests."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified in Appendix H: '11th Gen Intel Core i7-1185G7 processor (3.00 GHz, 8 cores) with 32.0 GiB of RAM'. Runtime figures are provided across experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Finite hypothesis classes are weakly auditable for statistical parity with sample complexity O((18/epsilon^2) * log(8|F|/delta)).",
       "evidence": "Theorem 2 (Section 4.1), proved in Appendix E.1.",
       "supported": "strong"
     },
     {
       "claim": "The SP dimension provides a necessary and sufficient characterization of weak auditability: a concept class F is agnostic and realizably weak auditable if and only if SP(F) is finite.",
       "evidence": "Corollary 5 (Section 4.1), derived from Theorem 4. Proofs in Appendix E.2.",
       "supported": "strong"
     },
     {
       "claim": "The SP dimension is upper bounded by VC dimension (VC(F) >= SP(F)), meaning auditing statistical parity has lower information complexity than learnability.",
       "evidence": "Proposition 6 (Section 4.1) with proof from definition of VC and SP dimensions.",
       "supported": "strong"
     },
     {
       "claim": "Model classes with infinite VC dimension are not weakly or strongly auditable.",
       "evidence": "Proposition 9 (Section 4.2), proved in Appendix F.3.",
       "supported": "strong"
     },
     {
       "claim": "The EPO framework accurately estimates statistical parity and recovers the prospect class on COMPAS and Student Performance datasets, with estimation error decreasing monotonically with sample size.",
       "evidence": "Section 5.2, Table 1, Figures 4 and 6. COMPAS estimation error 2.33×10^-2, Student Performance 8.22×10^-3 at budget 1000.",
       "supported": "moderate"
     },
     {
       "claim": "The framework naturally extends to other auditing objectives including prediction error, learning stability, generalization error, and robust risk.",
       "evidence": "Appendix B defines loss functions for each property (Table 2) and proves risk minimization implies weak auditing (Propositions 10, 11, 12). However, no experiments validate these extensions.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical", "benchmark-eval"],
   "key_findings": "This paper introduces a PAC auditing framework for group fairness (statistical parity) under strategic model updates, where the model owner may change the model class while preserving certain properties. The key theoretical contribution is the SP dimension, a novel combinatorial complexity measure that characterizes the sample complexity of auditing statistical parity and is provably upper bounded by VC dimension. The paper establishes that finite hypothesis classes are both weakly and strongly auditable, while infinite VC classes are not. Numerical experiments on COMPAS and Student Performance datasets confirm that the EPO oracle accurately estimates statistical parity and the prospect class with low runtime overhead.",
   "red_flags": [
     {
       "flag": "No empirical baselines",
       "detail": "The numerical experiments only evaluate the authors' EPO algorithm without comparing against any existing auditing methods, including the approach of Yan and Zhang (2022) which is the closest prior work."
     },
     {
       "flag": "Minimal experimental validation",
       "detail": "Only two small datasets (COMPAS and Student Performance) with two strategic model types each are tested. The experimental section is thin relative to the theoretical claims, and no experiments validate the extensions to learning error, generalization error, or robust risk."
     },
     {
       "flag": "Missing experimental details",
       "detail": "Key experimental parameters are unspecified: MLP architecture, random forest configuration, number of sampled hypotheses for the finite approximation, data preprocessing steps, and scikit-learn version. This makes full reproduction difficult."
     },
     {
       "flag": "Anonymous repository link",
       "detail": "The code is hosted at an anonymous review URL (anonymous.4open.science), which may become unavailable after the review process. No permanent archive (e.g., Zenodo) is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Active fairness auditing",
       "authors": ["Tom Yan", "Chicheng Zhang"],
       "year": 2022,
       "relevance": "Closest prior work on PAC auditing of statistical parity under manipulation-proofness, which this paper directly extends."
     },
     {
       "title": "Distribution-specific auditing for subgroup fairness",
       "authors": ["Daniel Hsu", "Jizhou Huang", "Brendan Juba"],
       "year": 2024,
       "relevance": "Studies auditing of statistical parity under Gaussian distributions and demonstrates computational hardness of fairness verification."
     },
     {
       "title": "Auditing fairness by betting",
       "authors": ["Ben Chugg", "Santiago Cortes-Gomez", "Bryan Wilder", "Aaditya Ramdas"],
       "year": 2023,
       "relevance": "Alternative approach to fairness auditing using hypothesis testing and sequential methods."
     },
     {
       "title": "Under manipulations, are some AI models harder to audit?",
       "authors": ["Augustin Godinot", "Erwan Le Merrer", "Gilles Trédan", "Camilla Penzo", "François Taïani"],
       "year": 2024,
       "relevance": "Studies the difficulty of auditing ML models under strategic manipulations, directly related to the adversarial auditing setting of this paper."
     },
     {
       "title": "Preventing fairness gerrymandering: Auditing and learning for subgroup fairness",
       "authors": ["Michael Kearns", "Seth Neel", "Aaron Roth", "Zhiwei Steven Wu"],
       "year": 2018,
       "relevance": "Foundational work on auditing statistical parity through reduction to weak agnostic learning."
     },
     {
       "title": "A reductions approach to fair classification",
       "authors": ["Alekh Agarwal", "Alina Beygelzimer", "Miroslav Dudík", "John Langford", "Hanna Wallach"],
       "year": 2018,
       "relevance": "Influential fairness-aware ML framework connecting fair classification to cost-sensitive learning."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Referenced in the context of LLM fairness shifts over time, relevant to AI evaluation methodology."
     },
     {
       "title": "How is ChatGPT's behavior changing over time?",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Referenced for LLM behavioral drift over updates, motivating the paper's framework for auditing under model updates."
     },
     {
       "title": "FairSense: Long-term fairness analysis of ML-enabled systems",
       "authors": ["Yining She", "Sumon Biswas", "Christian Kästner", "Eunsuk Kang"],
       "year": 2025,
       "arxiv_id": "2501.01665",
       "relevance": "Recent work on longitudinal fairness analysis of ML systems, directly relevant to auditing fairness under model drift."
     },
     {
       "title": "Closing the AI accountability gap: Defining an end-to-end framework for internal algorithmic auditing",
       "authors": ["Inioluwa Deborah Raji", "Andrew Smart", "Rebecca N White", "Margaret Mitchell", "Timnit Gebru"],
       "year": 2020,
       "relevance": "Framework for algorithmic auditing practices, providing context for the practical challenges of ML fairness auditing."
     }
   ]
 }