ai-research-survey

scan.json (21300B)

---

 {
   "paper": {
     "title": "Audit Trails for Accountability in Large Language Models",
     "authors": [
       "Victor Ojewale",
       "Harini Suresh",
       "Suresh Venkatasubramanian"
     ],
     "year": 2026,
     "venue": "Manuscript submitted to ACM",
     "arxiv_id": "2601.20727",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link for the proof-of-concept library: https://github.com/victorojewale/audit-trail-PoC (footnote 1, Section 6)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The paper includes an illustrative audit trail excerpt in Appendix A but this is just a small example, not a research dataset. The PoC repository reportedly includes a small illustrative audit_trail.jsonl, but no research data per se is released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. The paper describes a Python library but does not mention requirements.txt, Dockerfile, dependency versions, or setup instructions beyond what is in the README."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions the PoC repository includes a README walkthrough (Section 6.4), but the paper itself does not contain step-by-step reproduction instructions. The README is referenced but its contents are not provided in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical/design paper with no empirical experiments that produce quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative empirical claims are made; the paper proposes a framework and proof-of-concept implementation without quantitative comparisons."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No empirical experiments are conducted; there are no effects to report sizes for."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No empirical study with samples is conducted. This is a theoretical/design paper."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted that would produce variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses related work (model cards, datasheets, MLOps tooling, HELM) but does not formally compare the proposed audit trail framework against these alternatives on any evaluation criteria."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No formal baseline comparison is conducted. Related work is discussed qualitatively but not evaluated against the proposed system."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The system is presented as a conceptual framework with a proof-of-concept; there is no multi-component empirical evaluation where ablations would apply."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative evaluation is performed; there are no metrics to report."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes a system for auditors and developers but includes no user study, expert evaluation, or usability assessment of the tool or framework. The paper itself acknowledges the need for future 'empirical studies [that] could examine how developers, auditors, and regulators actually engage with audit trail data' (Section 9)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No dataset-based evaluation is performed; this criterion is structurally inapplicable."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative evaluation is performed that could be broken down by categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Discussion) discusses epistemic limits ('audit trails provide chronological and structural traceability but not causal explanation'), scalability challenges, ethical tensions (surveillance risks, privacy concerns), and conditions under which audit trails would be insufficient."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No experiments are conducted, so there are no negative experimental results. While limitations are discussed, no empirical findings show what did not work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims three contributions: (1) a lifecycle framework, (2) a reference architecture, and (3) a Python implementation. All three are presented in Sections 4, 5, and 6 respectively. The abstract does not overclaim empirical validation."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make causal claims. It proposes a framework and describes what audit trails 'can support' and 'would' enable, using conditional and future-oriented language throughout."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is clear that the PoC 'implements a representative subset of the reference architecture' (Section 6), is 'intentionally not a full governance platform' (Section 6), and future work is needed for empirical evaluation and adoption studies (Section 9)."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents no empirical results for which alternative explanations would need to be considered. It is a design/theoretical contribution."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any LLM. It proposes a framework for auditing LLM systems but does not itself use or test specific models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this paper. It is a framework/tool design paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No model training or evaluation experiments are conducted that would involve hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used or evaluated in this paper."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data is collected or preprocessed for empirical analysis. This is a theoretical/design paper."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Discussion) contains substantive discussion of limitations including epistemic limits, scalability challenges, complex system composition issues, and ethical/institutional tensions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific limitations: audit trails cannot establish causation (epistemic limits), scaling to millions of records poses practical challenges, multi-component architectures require synchronized schemas, and logs could be misused for surveillance."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states the PoC 'is intentionally not a full governance platform' and 'implements a representative subset of the reference architecture' (Section 6). Section 8 notes audit trails provide traceability 'but not causal explanation.' Section 9 explicitly calls out what the paper does not show: no empirical user studies, no field experiments, no differential privacy implementation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "No empirical data is collected. This is a theoretical/design paper with a proof-of-concept implementation."
       },
       "data_collection_described": {
         "applies": false,
         "answer": false,
         "justification": "No data collection is performed. The paper proposes a framework and implements a PoC library."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No participants or data subjects are involved. This is a design paper."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline exists for this paper's own methodology. The paper describes how audit trail data pipelines would work in practice, but this is the contribution itself, not a research methodology."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are identified with their affiliation at the Department of Computer Science, Brown University, USA."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of any funding disclosure means this criterion cannot be satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model on a benchmark. It proposes a framework and tool for audit trails."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical/design paper proposing a framework. It does not run experiments with LLM inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical/design paper. No significant computation was performed beyond implementing a small Python library."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM audit trails can serve as a sociotechnical mechanism for continuous accountability by linking technical provenance with governance records in a chronological, tamper-evident ledger.",
       "evidence": "The paper presents a conceptual framework (Section 4), reference architecture (Section 5), and proof-of-concept implementation (Section 6) to demonstrate feasibility, but provides no empirical validation of effectiveness.",
       "supported": "moderate"
     },
     {
       "claim": "The proposed audit trail framework can be integrated into existing LLM workflows with minimal code changes.",
       "evidence": "Section 6 describes a Python library (llm-audit-trail) with a HuggingFace TrainerCallback, FastAPI middleware, and governance CLI. Code snippets are shown in Figures 2-4. However, 'minimal' is not quantified and no integration cost study is conducted.",
       "supported": "moderate"
     },
     {
       "claim": "Existing documentation approaches (model cards, datasheets, MLOps tooling) leave an accountability gap because they are static snapshots or lack governance context.",
       "evidence": "Section 3 argues that model cards are 'curated and static' and MLOps tools are 'oriented toward developer needs, not governance.' The argument is based on analysis of existing tools rather than empirical measurement of the gap.",
       "supported": "moderate"
     },
     {
       "claim": "Hash-chain integrity verification makes tampering or deletion detectable in the audit trail.",
       "evidence": "Section 6.2 describes SHA-256 hash chaining where each event's curr_hash covers the serialized payload plus the previous hash. Section 6.3 describes a verify_log function. This is a standard cryptographic technique and the claim is well-grounded.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "theoretical"
   ],
   "key_findings": "The paper proposes LLM audit trails as a reusable accountability layer consisting of three components: a lifecycle framework specifying what to log across pretraining, adaptation, deployment, and monitoring; a three-layer reference architecture (Capture, Store, Use) with emitter-based event collection, append-only tamper-evident storage, and auditor-facing query tools; and a proof-of-concept Python library demonstrating integration with HuggingFace Trainer, FastAPI, and a governance CLI. The work is entirely theoretical and design-oriented, with no empirical evaluation of the framework's effectiveness, adoption costs, or impact on accountability outcomes.",
   "red_flags": [
     {
       "flag": "No empirical evaluation",
       "detail": "The paper proposes a framework and implements a proof-of-concept but conducts no empirical evaluation whatsoever -- no user studies, no case deployments, no measurement of adoption costs, integration overhead, or actual accountability improvements. The motivating scenarios in Section 2 are hypothetical, and Section 7 revisits them only to describe how the audit trail 'would' work, not how it actually performed."
     },
     {
       "flag": "Feasibility claims without evidence",
       "detail": "The paper claims the audit trail can be integrated with 'minimal code changes' and 'modest overhead' (Section 6.5, Section 9) but provides no measurements of integration effort, runtime overhead, storage costs, or developer experience. These are qualitative assertions about the PoC, not empirical findings."
     }
   ],
   "cited_papers": [
     {
       "title": "AI auditing: The Broken Bus on the Road to AI Accountability",
       "authors": ["Abeba Birhane", "Ryan Steed", "Victor Ojewale", "Briana Vecchione", "Inioluwa Deborah Raji"],
       "year": 2024,
       "doi": "10.1109/SaTML59370.2024.00037",
       "relevance": "Directly addresses AI auditing gaps and accountability infrastructure, relevant to evaluating methodology of AI safety/governance research."
     },
     {
       "title": "Towards AI Accountability Infrastructure: Gaps and Opportunities in AI Audit Tooling",
       "authors": ["Victor Ojewale", "Ryan Steed", "Briana Vecchione", "Abeba Birhane", "Inioluwa Deborah Raji"],
       "year": 2025,
       "doi": "10.1145/3706598.3713301",
       "relevance": "Surveys AI audit tooling landscape, directly relevant to the survey's scope on AI governance and accountability tooling."
     },
     {
       "title": "Auditing large language models: a three-layered approach",
       "authors": ["Jakob Mökander", "Jonas Schuett", "Hannah Rose Kirk", "Luciano Floridi"],
       "year": 2024,
       "doi": "10.1007/s43681-023-00289-2",
       "relevance": "Proposes a layered LLM audit framework (governance, model, application levels), directly relevant to AI safety and evaluation methodology."
     },
     {
       "title": "Closing the AI Accountability Gap: Defining an End-to-End Framework for Internal Algorithmic Auditing",
       "authors": ["Inioluwa Deborah Raji", "Andrew Smart", "Rebecca N. White", "Margaret Mitchell", "Timnit Gebru", "Ben Hutchinson", "Jamila Smith-Loud", "Daniel Theron", "Parker Barnes"],
       "year": 2020,
       "doi": "10.1145/3351095.3372873",
       "relevance": "Defines an end-to-end internal algorithmic auditing framework, foundational work for AI accountability research."
     },
     {
       "title": "Holistic Evaluation of Language Models",
       "authors": ["Percy Liang", "Rishi Bommasani", "Tony Lee"],
       "year": 2023,
       "relevance": "HELM benchmark and evaluation methodology for LLMs, directly relevant to the survey's scope on evaluation methodology and rigor."
     },
     {
       "title": "On the Opportunities and Risks of Foundation Models",
       "authors": ["Rishi Bommasani", "Drew A. Hudson"],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Foundational survey on foundation model risks and opportunities, relevant to the survey's scope on AI safety and capability assessment."
     },
     {
       "title": "Ecosystem Graphs: Documenting the Foundation Model Supply Chain",
       "authors": ["Rishi Bommasani", "Dilara Soylu", "Thomas I. Liao", "Kathleen A. Creel", "Percy Liang"],
       "year": 2024,
       "doi": "10.1609/aies.v7i1.31629",
       "relevance": "Maps the foundation model supply chain ecosystem, relevant to understanding AI development accountability and methodology."
     },
     {
       "title": "AI Supply Chains: An Emerging Ecosystem of AI Actors, Products, and Services",
       "authors": ["Aspen Hopkins", "Sarah H. Cen", "Andrew Ilyas", "Isabella Struckman", "Luis Videgaray", "Aleksander Mądry"],
       "year": 2025,
       "arxiv_id": "2504.20185",
       "relevance": "Characterizes the AI supply chain ecosystem, relevant to understanding distributed accountability in AI development."
     },
     {
       "title": "On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?",
       "authors": ["Emily M. Bender", "Timnit Gebru", "Angelina McMillan-Major", "Shmargaret Shmitchell"],
       "year": 2021,
       "doi": "10.1145/3442188.3445922",
       "relevance": "Influential paper on LLM risks and limitations, relevant to the survey's AI safety scope."
     },
     {
       "title": "OLMo: Accelerating the Science of Language Models",
       "authors": ["Dirk Groeneveld", "Iz Beltagy", "Pete Walsh"],
       "year": 2024,
       "arxiv_id": "2402.00838",
       "relevance": "Open language model with documented training provenance, relevant as an example of transparency practices in LLM development."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenges claims about emergent LLM abilities, relevant to evaluation methodology and claims about LLM capabilities."
     }
   ]
 }