ai-research-survey

scan.json (24517B)

---

 {
   "paper": {
     "title": "Analysis and Evaluation of Synthetic Data Generation in Speech Dysfluency Detection",
     "authors": ["Jinming Zhang", "Xuanru Zhou", "Jiachen Lian", "Shuhe Li", "William Li", "Zoe Ezzes", "Rian Bogley", "Lisa Wauters", "Zachary Miller", "Jet Vonk", "Brittany Morin", "Maria Gorno-Tempini", "Gopala Anumanchipalli"],
     "year": 2025,
     "venue": "arXiv (submitted to Interspeech)",
     "arxiv_id": "2505.22029",
     "doi": "10.48550/arXiv.2505.22029"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'All data, models, and code are open-sourced at https://github.com/Berkeley-Speech-Group/LLM-Dys.' A specific GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract explicitly states 'All data, models, and code are open-sourced' and points to the GitHub repository. The LLM-Dys dataset is the main contribution of the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency versions are mentioned in the paper. The paper mentions using VITS, Whisper-large-v3-turbo, and E2-TTS but does not provide setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The paper refers to the GitHub repository but does not include a 'Reproducing Results' section or specific commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-5 are reported as point estimates (e.g., F1=0.91, Precision=0.95) with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'state-of-the-art' performance and compares against baselines (Wagner et al., Yolo-Stutter, StutterNet) by simply comparing numbers. No statistical significance tests are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw metric values but does not provide effect sizes such as Cohen's d or relative improvements with baseline context. Comparisons between systems are presented as raw numbers in tables without quantifying the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Key evaluation sample sizes are mentioned (300 unique utterances per type for LLM-Dys testing, 200 manually annotated samples for SEP-28k, 80/120 for UCLASS) but none are justified or supported by power analysis. The choice of 200 manual annotations from a 28,000-clip dataset is not explained."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experimental results. It appears all results are from single runs with no indication of variability across seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against Wagner et al. [11], Yolo-Stutter [14], and StutterNet [1] in Tables 3 and 5. Cross-dataset audio quality comparisons include VCTK-token, VCTK, LibriTTS, and LibriStutter in Fig. 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are contemporary: Wagner et al. (Interspeech 2024), Yolo-Stutter (Interspeech 2024), and VCTK-Token (2024). StutterNet (2024) is also recent. These represent the current state of the art in dysfluency detection."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation-like studies: scaling law experiments (Section 4.3.3) showing impact of dataset size, and analysis of fluent-to-disfluent speech ratio (Section 4.3.4). However, these focus on data characteristics rather than model components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: Precision, Recall, F1-score, Accuracy, Token Error Rate (TER), and Token Distance (TD), reported across Tables 2-5."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper relies on Meta Audiobox Aesthetics (an automated tool) for quality evaluation and standard detection metrics. No human evaluation of the synthetic speech quality or detection results is conducted, despite claims about 'naturalness' which is inherently a subjective quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Testing on SEP-28k uses 200 manually annotated samples separate from training (zero-shot evaluation). UCLASS uses 120 test samples separate from the 80 fine-tuning samples. The LLM-Dys test set uses 300 utterances per type from a designated testing set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dysfluency type (Insertion, Repetition, Pause, Prolongation) and by level (word-level vs phoneme-level) across all tables. Fig. 2-3 also show per-type comparisons."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis are discussed. The paper does not show examples where the system fails or analyze systematic patterns in errors. Section 4.3.3 notes performance plateaus but does not analyze why."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3.3 reports that 'simply increasing textual data or diversity may not yield additional performance improvements' and shows performance plateaus or slight declines beyond certain dataset sizes. Section 2.2 notes VITS 'shows limitations in synthesizing fillers like um and uh.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'state-of-the-art performance' which is supported by Tables 3 and 5 showing improvements over baselines. The claim of 'most comprehensive dysfluent speech corpus' is supported by Table 1 (12,790 hours, 11 categories). The claim about LLM-enhanced quality is supported by Fig. 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'LLM-Dys achieves superior synthesis quality compared to other text-diversity-constrained simulation corpora' and attributes performance to 'consistent patterns in LLM-generated dysfluencies and our standardized TTS pipeline.' However, no controlled experiments isolate the specific causal factors — the LLM-based text generation, the TTS model choice, and the scale are confounded."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only in English with VCTK speakers and specific TTS models, yet the title and abstract make broad claims about 'speech dysfluency detection' without bounding to English or the specific speaker set. The conclusion mentions 'cross-lingual coverage' as future work, implicitly acknowledging this gap, but the main claims are not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed results. The improvement could be due to scale (12,790 hours vs smaller baselines), TTS model choice, LLM text quality, or speaker diversity, but these factors are not disentangled or discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'claude-3-5-sonnet' for text generation and 'Whisper-large-v3-turbo' for detection, but does not provide snapshot dates or API versions for Claude. The VITS and E2-TTS implementations lack version specifics."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The prompts we used can be found at our open-sourced page' but does not include the actual prompt text in the paper or appendix. Only a reference to the GitHub repository is given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters are largely unreported. The paper mentions pause durations (0.8-3.5s for word-level, 0.3-1.5s for phoneme-level) and prolongation extension (0.17-0.8s), but does not report LLM temperature/sampling settings, Whisper fine-tuning hyperparameters (learning rate, epochs, batch size), or TTS model parameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The LLM (Claude) is used for single-turn text generation, not as an agent."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the data pipeline: LLM text generation (Section 2.1), speech synthesis via VITS/E2-TTS (Section 2.2), special handling for pause and prolongation markers, and speaker variation (109 VCTK speakers). The processing steps from text to final audio are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion briefly mentions future directions (expanding styles, languages, articulatory priors) but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. There is no mention of the synthetic-to-real domain gap, potential biases in LLM-generated text patterns, or limitations of using automated quality metrics instead of human evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not note that results are specific to English, VCTK speakers, or the particular TTS models used. The conclusion mentions future cross-lingual work but does not frame current results as limited to the tested setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states all data are open-sourced at the GitHub repository. The entire LLM-Dys dataset (12,790 hours of synthetic speech) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the full data generation process: LLM-based text generation (Section 2.1), TTS synthesis (Section 2.2), with details on speaker variation, pause/prolongation handling, and statistics (Section 2.3)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The dataset is entirely synthetic, generated from LLMs and TTS models using existing speech corpora (VCTK)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented in Section 2 and Figure 1: LLM generates dysfluent text with labels, text is fed to TTS (VITS or E2-TTS), pause/prolongation markers are processed, and 109 speaker variants are generated per utterance. Table 1 provides counts at each level."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Acknowledgements) lists funding support: 'UC Noyce Initiative, Society of Hellman Fellows, NIH/NIDCD, and the Schwab Innovation fund.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Zhejiang University (China), UC Berkeley (USA), and UCSF (USA). The paper does not evaluate a commercial product from any of these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (UC Noyce Initiative, NIH/NIDCD, Hellman Fellows, Schwab Innovation fund) are academic/government research funders with no direct financial stake in the dysfluency detection results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. The paper uses Anthropic's Claude for text generation but does not disclose whether any authors have financial relationships with Anthropic."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Whisper-large-v3-turbo as the base model for fine-tuning and evaluates on SEP-28k and UCLASS benchmarks. No training data cutoff is stated for the Whisper model, making it impossible to assess whether benchmark data could have been in Whisper's pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The paper fine-tunes Whisper on synthetic data and tests on real data (SEP-28k, UCLASS), but does not discuss whether Whisper's pre-training data might include speech from VCTK speakers or overlap with test samples."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SEP-28k (2021) and UCLASS (2009) were both available before Whisper's training. The paper does not address whether Whisper may have been exposed to these datasets during its large-scale pre-training on web audio."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses synthetic data generation and automated evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The 'speakers' are from the VCTK corpus, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported. The system uses Claude API calls for text generation and runs TTS models for 12,790 hours of synthesis, but no costs are mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. Generating 12,790 hours of synthetic speech likely required substantial compute, and fine-tuning Whisper-large-v3-turbo is also non-trivial, but neither the GPU hours, training time, nor API costs are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-Dys is the most comprehensive dysfluent speech corpus with over 10,000 hours covering 11 dysfluency categories.",
       "evidence": "Table 1 shows 12,790 total hours across 11 types at word and phoneme levels. Comparison with prior datasets in Section 1 shows no prior dataset at this scale.",
       "supported": "strong"
     },
     {
       "claim": "LLM-Dys achieves superior synthesis quality compared to other simulation corpora and is comparable to real fluent speech.",
       "evidence": "Fig. 2 shows Meta Audiobox Aesthetics scores (CE, CU, PQ) where LLM-Dys scores are close to or exceed those of VCTK and LibriTTS (real speech corpora). However, evaluation is entirely automated — no human perceptual evaluation.",
       "supported": "moderate"
     },
     {
       "claim": "The system achieves state-of-the-art performance on real-world dysfluency detection benchmarks (SEP-28k and UCLASS).",
       "evidence": "Table 3 shows higher F1 scores than Wagner et al. and Yolo-Stutter on most dysfluency types for SEP-28k. Table 5 shows improved accuracy (0.971) and F1 (0.977) vs StutterNet (0.932) on UCLASS. However, comparisons are incomplete — Yolo-Stutter only reports recall, and the evaluation is zero-shot for SEP-28k.",
       "supported": "moderate"
     },
     {
       "claim": "Simply increasing textual data or diversity may not yield additional performance improvements unless a high-quality TTS model is employed.",
       "evidence": "Section 4.3.3 and Fig. 5 show performance plateaus after 3x4000 samples, with marginal or declining improvements at 3x12000. However, this finding is limited to the specific TTS model used and the claim about TTS quality is not experimentally verified.",
       "supported": "weak"
     },
     {
       "claim": "The optimal fluent-to-disfluent speech ratio is approximately 0.05.",
       "evidence": "Section 4.3.4 and Fig. 5 (right) show performance peaks at ratio ~0.05. But this is specific to the SEP-28k disfluency distribution and is not generalizable.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces LLM-Dys, a large-scale (12,790 hours) synthetic dysfluency speech dataset covering 11 categories at both word and phoneme levels, generated using Claude 3.5 Sonnet for text simulation and VITS/E2-TTS for speech synthesis. The system achieves state-of-the-art performance on SEP-28k and UCLASS benchmarks for dysfluency detection using a fine-tuned Whisper-large-v3-turbo model. Scaling experiments show diminishing returns beyond 3x4000 training samples per type, and the optimal fluent-to-disfluent ratio is approximately 0.05. Quality evaluation via Meta Audiobox Aesthetics shows the synthetic speech approaches real speech quality.",
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across Tables 2-5 are single point estimates with no confidence intervals, standard deviations, or indication of variability across runs. For a machine learning paper, this makes it impossible to assess whether differences are meaningful."
     },
     {
       "flag": "No human evaluation of speech quality",
       "detail": "The paper makes strong claims about 'naturalness' and 'quality' of synthetic speech but relies entirely on an automated metric (Meta Audiobox Aesthetics). Speech naturalness is inherently perceptual and subjective, making human evaluation critical for such claims."
     },
     {
       "flag": "Incomplete baseline comparisons",
       "detail": "Comparisons with Yolo-Stutter are recall-only (missing precision and F1). Wagner et al. comparison only shows F1. The metrics are not consistent across baselines, making fair comparison difficult."
     },
     {
       "flag": "Confounded experimental design",
       "detail": "The improvement over prior work could be due to multiple confounded factors: LLM text generation quality, larger dataset scale (12,790 hrs vs smaller baselines), TTS model choice (VITS/E2-TTS), or speaker diversity (109 speakers). No controlled experiment isolates which factor drives the improvement."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries. Generalization claims are unbounded — results are specific to English with VCTK speakers and particular TTS models, but the paper does not acknowledge this."
     },
     {
       "flag": "Small manually annotated test sets",
       "detail": "The SEP-28k evaluation uses only 200 manually annotated samples from a 28,000-clip dataset. The UCLASS evaluation uses 120 test samples with only 80 for fine-tuning. These small sample sizes are not justified and limit the reliability of the reported metrics."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for dysfluency detection in stuttered speech",
       "authors": ["D. Wagner", "S. P. Bayerl", "I. Baumann", "K. Riedhammer", "E. Nöth", "T. Bocklet"],
       "year": 2024,
       "relevance": "Uses LLMs for speech dysfluency detection, directly relevant to LLM capabilities in speech analysis tasks."
     },
     {
       "title": "Yolo-stutter: End-to-end region-wise speech dysfluency detection",
       "authors": ["X. Zhou", "A. Kashyap", "S. Li"],
       "year": 2024,
       "relevance": "End-to-end dysfluency detection baseline, representative of current approaches in the field."
     },
     {
       "title": "Ssdm: Scalable speech dysfluency modeling",
       "authors": ["J. Lian", "X. Zhou", "Z. Ezzes"],
       "year": 2024,
       "relevance": "NeurIPS 2024 paper on scalable dysfluency modeling using TTS-based simulation, directly related methodology."
     },
     {
       "title": "Robust speech recognition via large-scale weak supervision",
       "authors": ["A. Radford", "J. W. Kim", "T. Xu", "G. Brockman", "C. McLeavey", "I. Sutskever"],
       "year": 2023,
       "relevance": "Whisper model used as the base for the detection framework, relevant to LLM/foundation model evaluation methodology."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["J. Kaplan", "S. McCandlish", "T. Henighan", "T. B. Brown"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Referenced for the scaling law experiments; relevant to understanding data scaling in AI systems."
     },
     {
       "title": "Time and tokens: Benchmarking end-to-end speech dysfluency detection",
       "authors": ["X. Zhou", "J. Lian", "C. J. Cho"],
       "year": 2024,
       "arxiv_id": "2409.13582",
       "relevance": "VCTK-Token benchmark and token-based detection framework that this paper builds upon."
     },
     {
       "title": "Meta audiobox aesthetics: Unified automatic quality assessment for speech, music, and sound",
       "authors": ["A. Tjandra", "Y.-C. Wu", "B. Guo"],
       "year": 2025,
       "arxiv_id": "2502.05139",
       "relevance": "Automated audio quality evaluation tool used as the primary quality metric, relevant to evaluation methodology."
     },
     {
       "title": "F5-tts: A fairytaler that fakes fluent and faithful speech with flow matching",
       "authors": ["Y. Chen", "Z. Niu", "Z. Ma"],
       "year": 2024,
       "arxiv_id": "2410.06885",
       "relevance": "TTS model variant (E2-TTS) used for word-level insertion synthesis in the pipeline."
     }
   ]
 }