ai-research-survey

scan.json (30827B)

---

 {
   "paper": {
     "title": "HearSay Benchmark: Do Audio LLMs Leak What They Hear?",
     "authors": [
       "Jin Wang",
       "Liang Lin",
       "Kaiwen Luo",
       "Weiliu Wang",
       "Yitian Chen",
       "Moayad Aloqaily",
       "Xuehai Tang",
       "Zhenhong Zhou",
       "Kun Wang",
       "Li Sun",
       "Qingsong Wen"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.03783",
     "doi": "10.48550/arXiv.2601.03783"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ALLMs inherently extract private attributes from voiceprints, achieving 92.89% average accuracy on gender across 13 models, while exhibiting near-zero refusal rates for physiological attributes. Chain-of-Thought prompting amplifies privacy risks in capable models (22.1% boost on accent inference for Qwen3-Omni-Flash) but degrades performance in weaker models. Capable models leverage genuine acoustic evidence to correct statistical biases, while weaker models hallucinate based on prior distributions. Lightweight prompt defense improves refusal rates overall but remains ineffective for deeply ingrained physiological inferences like gender.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in abstract: https://github.com/JinWang79/HearSay_Benchmark. 'The codes and dataset are available at' this URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset is available through controlled access at the GitHub URL, plus external public datasets (NISP, VocalSound, VoxCeleb2/Age-Vox-Celeb) are all publicly available. Appendix A.2 describes licensing terms."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'four H20 GPUs' for inference but provides no requirements.txt, Dockerfile, conda environment, or library version details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. No README with commands or a 'Reproducing Results' section is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates (IAR%, ARR%) with no confidence intervals, error bars, or ± notation anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'surpasses random baselines by 46.4%', 'CoT boosts Accent inference accuracy by 22.1%') based solely on comparing numbers without any statistical tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context throughout: Figure 3 shows 46.4% above random guessing baseline with both values visible, Table 1 shows absolute IAR/ARR values for all models, and CoT improvements are given with both before and after values (e.g., Accent 'surges by 22.1%')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset comprises 22,064 audio clips but no justification is given for this sample size. No power analysis or discussion of whether this size is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Random Guessing Baseline is included (Figure 3). Additionally, a Transcribed-Text control setting isolates semantic from acoustic features. 13 models are compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Evaluated models include recent 2024-2025 releases: Qwen3-Omni-Flash, Gemini-2.5-Pro, GPT-4o-Audio, Step-Audio-2, Kimi-Audio. These represent the current state of the art in audio LLMs."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation-like comparisons: With-Audio vs Transcribed-Text (Section 4.2), direct answering vs CoT prompting (Section 4.3), base vs defense prompt (Section 4.5). Each isolates a specific factor."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three distinct metrics are defined and used: Inference Accuracy Rate (IAR), Answer Refusal Rate (ARR), and Blind Bias Rate (BBR), each measuring different aspects of privacy risk."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation of model outputs is entirely automated using GPT-4o-mini as judge (Section 4.1). Human verification was used only for dataset construction (Stage III), not for evaluating model outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HearSay is a newly constructed benchmark that the evaluated models were not fine-tuned on. The entire benchmark serves as a held-out test set since no model was trained or tuned on this data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-attribute breakdowns across all 8 privacy dimensions for each of 13 models. Figure 4 gives per-attribute CoT comparisons. Table 2 shows per-attribute defense effectiveness."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses models that fail to use acoustic evidence (MERaLION, Kimi-Audio 'cluster tightly around the diagonal line'). Section 4.3 discusses CoT hurting models (Kimi-Audio Income drops 13.7%). Section 4.5 shows defense failure on Gender."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "CoT degrades performance for Kimi-Audio and Qwen2.5-Omni (Obs 3). Prompt defense is 'largely ineffective against deeply ingrained physiological inferences' with Gender ARR remaining near 0% (Obs 8). MERaLION shows 'limited responsiveness even to general defense prompts.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are verified in results: 92.89% gender accuracy (Table 1 average), near-zero refusal rates (Table 1 ARR columns), CoT boosts accent by 22.1% (Figure 4 Qwen3-Omni-Flash panel). All specific numbers in the abstract match the experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — 'CoT reasoning exacerbates privacy risks' — is supported by controlled comparison (same models, same data, with/without CoT). The Transcribed-Text control (Section 4.2) isolates voiceprint from semantic content through controlled manipulation. These are adequate single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title and abstract frame findings about 'Audio LLMs' broadly, but the benchmark is English-centric (acknowledged only in Limitations Section 6) and speakers are predominantly from academic/professional backgrounds. The paper generalizes from 13 specific models to 'ALLMs' throughout without adequately bounding this."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 (RQ3) directly addresses the alternative explanation that models hallucinate based on statistical priors rather than using acoustic evidence, using BBR analysis and Figure 5. Section 4.2 addresses semantic leakage as an alternative explanation using the Transcribed-Text control."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures IAR (accuracy of attribute inference from audio) and frames this as 'privacy leakage.' The measurement closely matches the claim — successfully inferring private attributes from voiceprints IS the privacy leak being studied. No significant proxy gap exists between measurement and framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Some models have size specifications (e.g., 'Qwen2-Audio-7B-Instruct', 'Qwen2.5-Omni-7B') but others use marketing names without snapshot dates: 'GPT-4o-Audio', 'Gemini-2.5-Flash', 'Gemini-2.5-Pro'. No API version or snapshot date is provided for any closed-source model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for all experimental conditions is provided in Appendix C: Profiling ALLM prompt, Privacy Judge prompt, Transcribed-Text control prompt, Chain-of-Thought profiling prompt, and Defense prompt. Actual text, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for any of the 13 models or for the GPT-4o-mini judge. Only hardware is mentioned ('four H20 GPUs')."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models receive audio clips and prompts directly for inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes a four-stage data curation pipeline: (I) profiling with confidence tags, (II) audio extraction from lectures, (III) human review discarding 'inferred' attributes, (IV) supplementation with external datasets for dynamic traits. Each stage's purpose and filtering logic is described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitation' is a dedicated section with substantive discussion of three specific limitation areas."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 identifies specific threats: (1) defense exploration limited to textual prompt engineering, not audio-level defenses like encoder-stage de-identification, (2) dataset is predominantly English-centric limiting cross-lingual generalization, (3) evaluation limited to standard inference and CoT, excluding jailbreaking and adversarial audio perturbations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what was NOT tested: audio-level defenses, multilingual corpora for cross-lingual privacy risks, and aggressive attack vectors including privacy-targeted jailbreaking and adversarial audio perturbations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset available through controlled access mechanism (Appendix A.2). External datasets (NISP, VocalSound, VoxCeleb2) are publicly available. 'HearSay is available exclusively through a controlled access mechanism, requiring researchers to submit a formal application.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 details the four-stage collection pipeline with time period consideration (Stage IV addresses time-sensitive attributes), instruments used (profiling agent with confidence tags), and inclusion/exclusion criteria (only 'verified' attributes retained, 'inferred' discarded)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states speakers come from 'public lecture videos' and external datasets but does not describe how specific lectures or speakers were selected from the pool of available sources. No selection criteria for which public lectures were included, which could introduce demographic and socioeconomic bias toward academic/professional speakers."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The four-stage pipeline (Section 3.2) documents each transformation: profiling → audio extraction → human verification → external supplementation. Section 3.3 reports final statistics (22,064 clips). Figure 2 shows distribution across attributes and categories."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant disclosures appear anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: XDU, NTU, NCEPU, BUPT, SHU, UAEU, UCAS-IIE, Squirrel AI. None of the authors are affiliated with the companies whose models are being evaluated (OpenAI, Google, etc.)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. Absence of disclosure is not evidence of absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper. One author is from 'Squirrel AI' (an AI company) but no financial interests declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 13 evaluated models, despite benchmark audio clips coming from public sources that models may have encountered during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses audio from public lectures and public datasets (VoxCeleb2 published 2018, NISP 2021, VocalSound 2022). Models trained after these dates may have encountered this audio. No discussion of potential overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HearSay uses audio from publicly available datasets and lectures that pre-date most evaluated models' training. No contamination analysis is performed despite the risk that models may have been exposed to these audio clips or related speaker information during pre-training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation. Speakers in audio clips are not study participants — they are subjects of public recordings used to construct a benchmark."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper includes an ethical statement (Section 7) addressing data ethics and dual-use risks but no IRB approval is needed or mentioned."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Speaker demographics are the attributes being predicted, not participant demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Data inclusion criteria for the benchmark are covered under data_integrity."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental randomization required for benchmark evaluation."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Blinding is not applicable to automated benchmark evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. No attrition concept applies to benchmark evaluation."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or per-example latency reported despite evaluating 13 models across 22,064 clips × 8 attributes (over 175,000 inference calls minimum)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only 'four H20 GPUs' is mentioned for open-source model inference. No total GPU hours, API spend, or training time is stated. Closed-source API costs are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results reported. All results appear to be from single runs without any seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated for the main evaluation (Table 1). BBR uses N independent trials (Equation 6) but the specific N value is not reported, and the main IAR/ARR results don't specify run count."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Sampling parameters (temperature, top-p) are not even disclosed, let alone any search over them."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how configuration choices were made for each model. Default settings appear to be used but this is not stated or justified."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 13 models × 8 attributes but performs no statistical tests at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the HearSay benchmark and evaluate models on it. No acknowledgment of potential bias in benchmark design favoring their conclusions. No independent evaluation or discussion of author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes and compute requirements are compared (7B open-source vs large closed-source APIs) without any discussion of compute budget differences or matched-compute comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes two construct validity checks: (1) BBR analysis (Section 4.4) verifying models use acoustic evidence rather than statistical priors, and (2) Transcribed-Text control (Section 4.2) confirming privacy leakage stems from voiceprints rather than semantic content. Both directly validate that the benchmark measures what it claims."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models receive audio clips directly for inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark audio comes from public sources including VoxCeleb2 (2018), NISP (2021), and VocalSound (2022), all predating the evaluated models' training. No discussion of whether models encountered this audio during pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The profiling prompt explicitly instructs models to 'analyze a voice based ONLY on its physical and paralinguistic qualities' and specifies the target dimension. No discussion of whether the prompt itself cues models toward certain responses or leaks information about expected answers."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple audio clips may come from the same speaker (22,064 clips from an unspecified number of unique speakers, though 'broad spectrum of unique vocal identities' is mentioned). Non-independence between clips from the same speaker is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. No canary strings, membership inference tests, or decontamination pipelines are used to verify the benchmark data wasn't in model training sets."
       }
     }
   },
   "claims": [
     {
       "claim": "ALLMs inherently extract private attributes from voiceprints, with audio-based inference surpassing random baselines by 46.4% on average and achieving 92.89% accuracy on gender.",
       "evidence": "Table 1 shows average Gender IAR of 92.89% across open-source models. Figure 3 shows the 46.4% gap between With-Audio and Random Guessing Baseline across 8 attributes. Results from 13 models across 8 attributes.",
       "supported": "strong"
     },
     {
       "claim": "Current ALLMs exhibit near-zero refusal rates for privacy-intruding requests on physiological attributes, with only GPT-4o-Audio demonstrating appropriate defensive behavior.",
       "evidence": "Table 1 shows most open-source models have 0.00% ARR for Gender and other physiological attributes. GPT-4o-Audio achieves 100% ARR on Weight. Average ARR across open-source models is below 1% for Gender.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-Thought reasoning amplifies privacy risks in capable models, boosting Accent inference accuracy by 22.1% for Qwen3-Omni-Flash.",
       "evidence": "Figure 4 shows CoT boosts Qwen3-Omni-Flash on Accent by 22.1%, Education by 7.0%, and Income by 10.8%. However, the effect is mixed — Kimi-Audio's Income drops 13.7% and Qwen2.5-Omni's Weight drops 23.5% with CoT.",
       "supported": "moderate"
     },
     {
       "claim": "Capable models leverage acoustic evidence to correct inherent statistical biases, rather than hallucinating based on prior distributions.",
       "evidence": "Section 4.4 and Figure 5 show high-performing models (Qwen3-Omni-Flash, MiniCPM-o-2.6) fall in the 'Correction Zone' while weaker models (MERaLION, Kimi-Audio) cluster near the diagonal. BBR analysis across N trials with empty audio input.",
       "supported": "moderate"
     },
     {
       "claim": "Lightweight prompt defense significantly boosts refusal rates for responsive models but fails for physiological attributes, especially gender.",
       "evidence": "Table 2 shows MiniCPM-o-2.6 overall ARR rises from near-zero to 61.42% with defense prompt. However, Gender ARR remains near 0% across all models even with defense. Average improvement varies from +0.01% (Gender) to +72.2% (Income).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across Tables 1, 2 and Figures 3, 4 are single-run point estimates with no error bars, confidence intervals, variance, or any measure of uncertainty. With stochastic model outputs, results could vary substantially across runs."
     },
     {
       "flag": "Unvalidated LLM-as-judge methodology",
       "detail": "GPT-4o-mini is used as the automated judge for determining inference accuracy and refusal behavior, but no validation of judge agreement with human judgment is provided. The accuracy of the automated evaluation is assumed without verification."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Numerous comparative claims are made (model A outperforms model B, CoT improves/degrades performance, defense boosts refusal rates) based solely on comparing point estimates without any statistical tests to determine if differences are significant."
     },
     {
       "flag": "Ecological validity concerns",
       "detail": "The profiling prompts explicitly instruct models as 'Forensic Vocal Profilers' to analyze voice for private attributes. This differs substantially from natural user interactions and may overestimate real-world privacy risk. The attack model assumes adversarial prompting, which is acknowledged but the gap between benchmark and real-world privacy risk is not quantified."
     },
     {
       "flag": "Speaker demographic bias in benchmark construction",
       "detail": "Speakers are primarily drawn from public lecture videos, which overrepresents academic/professional speakers. The benchmark claims to be 'comprehensive' but this selection process creates demographic skew that could affect generalizability of privacy risk findings."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "Audio from public datasets (VoxCeleb2, NISP, VocalSound) and public lectures likely appears in training data of large multimodal models. Models may recognize specific speakers or audio clips rather than performing genuine acoustic inference. This is completely unaddressed."
     }
   ],
   "cited_papers": [
     {
       "title": "Beyond memorization: Violating privacy via inference with large language models",
       "authors": ["Robin Staab", "Mark Vero", "Mislav Balunović", "Martin T. Vechev"],
       "year": 2023,
       "arxiv_id": "2310.07298",
       "relevance": "Foundational work on LLM privacy inference showing models can infer personal attributes from unstructured text, directly motivating HearSay's extension to audio."
     },
     {
       "title": "Extracting training data from large language models",
       "authors": ["Nicholas Carlini", "Florian Tramer", "Eric Wallace"],
       "year": 2021,
       "relevance": "Seminal work on LLM memorization and privacy risks from training data extraction, establishing the broader privacy threat model for language models."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the foundation models whose audio variant (GPT-4o-Audio) is evaluated in this benchmark."
     },
     {
       "title": "AudioTrust: Benchmarking the Multifaceted Trustworthiness of Audio Large Language Models",
       "authors": ["Kai Li", "Can Shen", "Yile Liu"],
       "year": 2025,
       "arxiv_id": "2505.16211",
       "relevance": "Concurrent work on trustworthiness benchmarking for audio LLMs, establishing the broader safety evaluation landscape for ALLMs."
     },
     {
       "title": "A Comprehensive Survey in LLM (-Agent) Full Stack Safety: Data, Training and Deployment",
       "authors": ["Kun Wang", "Guibin Zhang", "Zhenhong Zhou"],
       "year": 2025,
       "arxiv_id": "2504.15585",
       "relevance": "Comprehensive survey of LLM and agent safety covering data, training, and deployment risks relevant to understanding privacy implications."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang"],
       "year": 2023,
       "relevance": "Establishes the LLM-as-judge evaluation paradigm used by HearSay for automated assessment of model inference accuracy and refusal behavior."
     },
     {
       "title": "Do-not-answer: A dataset for evaluating safeguards in LLMs",
       "authors": ["Yuxia Wang", "Haonan Li", "Xudong Han", "Preslav Nakov", "Timothy Baldwin"],
       "year": 2023,
       "arxiv_id": "2308.13387",
       "relevance": "Dataset for evaluating LLM safety refusal capabilities, directly related to HearSay's Answer Refusal Rate metric."
     },
     {
       "title": "A survey on code generation with LLM-based agents",
       "authors": ["Yihong Dong", "Xue Jiang"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey on LLM-based agent capabilities in code generation, part of the broader LLM capability assessment landscape."
     },
     {
       "title": "Gemini: a family of highly capable multimodal models",
       "authors": ["Gemini Team"],
       "year": 2023,
       "arxiv_id": "2312.11805",
       "relevance": "Technical report for the Gemini model family, whose 2.5 variants (Flash-Lite, Flash, Pro) are evaluated in the HearSay benchmark."
     },
     {
       "title": "Calibrate before use: Improving few-shot performance of language models",
       "authors": ["Zihao Zhao", "Eric Wallace", "Shi Feng", "Dan Klein", "Sameer Singh"],
       "year": 2021,
       "relevance": "Content-free calibration methodology that inspired HearSay's Blind Bias Rate metric for distinguishing genuine inference from statistical priors."
     },
     {
       "title": "Towards understanding chain-of-thought prompting: An empirical study of what matters",
       "authors": ["Boshi Wang", "Sewon Min", "Xiang Deng"],
       "year": 2023,
       "relevance": "Empirical analysis of CoT prompting mechanisms, relevant to HearSay's finding that CoT amplifies privacy risks in capable models."
     }
   ]
 }