ai-research-survey

scan.json (27637B)

---

 {
   "paper": {
     "title": "Linguistics Theory Meets LLM: Code-Switched Text Generation via Equivalence Constrained Large Language Models",
     "authors": [
       "Garry Kuwanto",
       "Chaitanya Agarwal",
       "Genta Indra Winata",
       "Derry Tanti Wijaya"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2410.22660",
     "doi": "10.48550/arXiv.2410.22660"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "EZSWITCH integrates Equivalence Constraint Theory (ECT) with LLMs (Llama3, Llama3.1, Aya23) to generate code-switched text, showing statistically significant fluency improvements over unconstrained baselines (p<.001) across Hindi-English, Tamil-English, and Malayalam-English. A comprehensive correlation study reveals existing automatic metrics (BLEU, COMET, BERTScore) correlate poorly with human judgments for code-switching (Kendall's tau ≤0.29), while GPT-4o-mini achieves ~0.5 correlation. Strong directional asymmetry was found: Indic-to-English code-switching significantly outperforms English-to-Indic (mean difference 0.47 accuracy, 0.40 fluency, p<.001).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code released at https://github.com/gkuwanto/ezswitch as stated in the abstract footnote."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "CSPREF human preference dataset released at https://huggingface.co/datasets/garrykuwanto/cspref. Input datasets (HinGE, Samanantar) are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Only hardware mentioned: 'single NVIDIA L40 GPU with 48GB of memory' (Section 3.1). No requirements.txt, Dockerfile, or library versions provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions found in the paper. The methodology is described but no runnable reproduction guide or README commands are mentioned."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and 3 report only point estimates (mean scores). No confidence intervals, error bars, or ± notation on any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "One-way ANOVAs conducted for Model, Method, and Direction factors (Table 6, all p<.001). Tukey's post-hoc tests with p-values reported for pairwise comparisons (Tables 7, 8, 9)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tukey's post-hoc tables (7, 8, 9) report mean differences alongside p-values, and Table 3 provides individual condition means, giving readers sufficient context to assess magnitude of effects (e.g., 'Llama3.1 vs. Aya23, Accuracy, 0.1881')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 150 samples were chosen for human evaluation, or why 2,000-2,766 parallel sentences were used per language pair. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations or variance measures reported for generation results. Inter-annotator agreement is reported via Krippendorff's alpha (Table 11) but no variance across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three methods compared: Baseline (unconstrained LLM), Human ECT (ECT with human translations), and EZSWITCH (ECT with LLM translations). Also compared with Word Replacement (WR) from Winata et al. (2019)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models used are contemporary (Llama3, Llama3.1, Aya23 — all 2024). WR baseline (Winata et al. 2019) is the standard prior method for ECT-based code-switching generation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The three-method comparison effectively ablates key components: Baseline removes ECT constraints entirely; Human ECT vs EZSWITCH isolates the effect of translation source (human vs LLM). Statistical tests (Tables 7, 8) quantify each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: Human Accuracy, Human Fluency, COMET (l1, l2, avg), GPT4oa, GPT4of, BLEU, BERTScore variants (Table 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Extensive human evaluation: 24,300 total evaluations by native bilingual speakers rating accuracy and fluency on a 1-3 scale, with 3 evaluators per sentence (Section 3.6)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No description of dev/test split. The 150-sample subset for human evaluation is randomly sampled from the full dataset, but there is no indication of a separate development set used for prompt or method tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by translation direction (English→CS vs Indic→CS), by model (Aya23, Llama3, Llama3.1), and by method (Baseline, Human ECT, EZSWITCH) in Tables 2 and 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses where EZSWITCH underperforms baseline (from Indic input). Section 5.3 discusses directional asymmetry and potential annotator bias. Section 5.4 discusses metric failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 reports that from Indic input, 'human evaluations show that EZSWITCH and Human ECT perform similarly, but both underperform compared to the Baseline.' This is a clear negative result for the proposed method."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'significant improvement in quality' which is supported by ANOVA (F=20.80, p<.001 for Method on Fluency) and Tukey's tests showing significant fluency improvements over baseline in both directions (Table 8). The paper appropriately hedges about direction-dependent effects in the body."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims 'incorporating linguistic constraints into LLMs leads to more robust and human-aligned generation.' The controlled comparison between Baseline (no ECT) and ECT-constrained methods, with statistical tests, provides adequate evidence for this causal claim through systematic single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Abstract claims results 'pav[e] the way for scalable code-switching text generation across diverse language pairs' but only three Indic-English pairs were tested. The title ('Code-Switched Text Generation') is broader than what was evaluated (three specific language pairs with 8B open-source models only)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses annotator L2 bias as an alternative explanation for directional asymmetry. Section 5.2 discusses LLM familiarity with Indic-English code-switching from training data as an alternative explanation for baseline performance."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy (meaning preservation) and fluency (naturalness) of code-switched text directly via human ratings. Claims match the granularity of measurements — they discuss code-switching generation quality in terms of these two specific dimensions, not broader unsubstantiated framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models specified with sizes (Aya23 8B, Llama3 8B, Llama3.1 8B) but GPT-4o-mini — used as a key evaluation tool — is specified only by marketing name without a snapshot date or API version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 (Appendix C) provides the actual prompt text for all methods: Translate, Baseline, EZSWITCH, and GPT Eval. Placeholders (e.g., <Input Sentence>) are clearly defined by the documented methodology."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, max tokens, or any sampling parameters for any of the LLMs (Llama3, Llama3.1, Aya23, GPT-4o-mini)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a linear pipeline: translation → alignment → prompt-based generation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 documents the full pipeline: obtaining translations (human or LLM, Section 2.3.1), bitext alignment with GIZA++ (Section 2.3.2), identification of valid switching points via ECT (Section 2.3.3 and Algorithm 1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present, discussing scope restrictions to open-source models, plans for commercial model extension, and intent to include more languages."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section contains generic statements: 'our study is intentionally limited in scope, focusing exclusively on the evaluation of open-source language models to ensure reproducibility.' No specific threats like sample size adequacy, annotator calibration concerns, or particular failure modes of ECT are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states the study is restricted to open-source models and three Indic-English language pairs, and that extension to commercial models and additional languages is future work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "CSPREF dataset released on HuggingFace (https://huggingface.co/datasets/garrykuwanto/cspref) contains human preference annotations for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 describes human evaluation setup in detail: 150 samples per language, 18 generation settings, 3 evaluators per sentence, 1-3 discrete scale, and what evaluators could see during annotation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Ethics Statement describes recruitment: 'DeccanAI ethically recruits human evaluators in India, and evaluates them to determine their English and native language proficiency via their crowdsourcing platform.' Evaluators are contractually employed and trained on annotation guidelines."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from parallel corpora → translation → alignment → switching points → generation → human evaluation is documented step by step in Section 2.3. Dataset sizes stated in Table 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure found anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed: Boston University, Deccan AI, Capital One, Monash University Indonesia. Note states 'The work was done outside the affiliation' for one author."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding not disclosed so independence cannot be assessed. Notably, co-author Chaitanya Agarwal is from Deccan AI, which also provided the annotation services used for evaluation — a potential conflict that is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the models (Llama3, Llama3.1, Aya23). These models could have been trained on the evaluation datasets."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Llama3/Aya23 training data includes HinGE or Samanantar datasets, which are public and could be in pre-training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HinGE (2021) and Samanantar (2022) are public datasets that predate Llama3 (2024) training. No discussion of whether the models may have seen these sentences during training, which could inflate generation quality."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration for the human evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Ethics Statement: 'All aspects of this research were reviewed and approved by the Institutional Review Board of our organization.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only broad characterization: evaluators are native Indic language speakers, proficient in English, from 'major cities in India.' No structured demographics (age, gender, education, years of bilingual experience)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Ethics Statement describes criteria: evaluators must be 'native speakers of the respective Indic languages they assess and are proficient in English. Their language proficiency is evaluated through custom online tests.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an annotation task, not an experimental study with treatment conditions. Evaluators all perform the same task (rating sentences), so randomization to conditions is not applicable."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No description of whether evaluators were blinded to which method/model generated each sentence. Evaluators see the input and output but blinding to experimental condition is not mentioned."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of evaluator attrition, dropout, or how many evaluators started versus completed the annotation task."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of inference cost, tokens consumed, wall-clock time per generation, or total API costs despite running 18 generation configurations across thousands of sentences."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware mentioned ('single NVIDIA L40 GPU with 48GB of memory') but no total compute budget, GPU hours, or runtime reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis for any generation experiments."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Not stated whether results are from a single generation run or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search described. Prompts appear to be designed without systematic tuning, and no search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 18 configurations (3 methods × 3 models × 2 directions) are reported in Tables 2 and 3. No selective reporting of best results; all conditions presented."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Tukey's HSD post-hoc test is used for pairwise comparisons (Tables 7, 8, 9), which inherently controls for family-wise error rate across multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment that the authors are evaluating their own system (EZSWITCH) against baselines. No independent evaluation or discussion of author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All methods use the same models with different prompts. Compute differences between methods are negligible."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 extensively discusses whether automatic metrics (BLEU, COMET, BERTScore) actually capture code-switching quality. Table 4 quantifies the gap between metrics and human judgment. The paper demonstrates that standard benchmarks fail to measure what they claim."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The pipeline is a direct prompting approach without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. HinGE (2021) and Samanantar (2022) predate the training of Llama3/Aya23 (2024), so temporal leakage is a real concern."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The EZSWITCH method provides target-language words as constraints in the prompt, which could be seen as providing information that aids generation quality, but this is by design rather than leakage. The broader question of evaluation setup leaking information is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether the public datasets (HinGE, Samanantar) appeared in LLM training data, which would create non-independence between training and evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method used. No membership inference, n-gram overlap analysis, or decontamination applied."
       }
     }
   },
   "claims": [
     {
       "claim": "ECT-constrained generation (EZSWITCH) produces significantly more fluent code-switched text than unconstrained baseline LLMs.",
       "evidence": "ANOVA: Method factor F=20.80, p<.001 for Fluency (Table 6). Tukey's tests show Human ECT and EZSWITCH both significantly outperform Baseline for Fluency in both directions (Table 8, p<.003 for all fluency comparisons).",
       "supported": "moderate"
     },
     {
       "claim": "Existing automatic metrics (BLEU, COMET, BERTScore) correlate poorly with human judgments for code-switching evaluation.",
       "evidence": "Table 4: Kendall's tau correlations with human accuracy/fluency: COMET_avg (0.246/0.290), BLEU (0.229/0.201), BERTScore variants (0.064-0.204). All below 0.3.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o-mini as evaluator achieves better alignment with human judgments than traditional metrics for code-switching.",
       "evidence": "Table 4: GPT4oa achieves 0.558 correlation with human accuracy and GPT4of achieves 0.514 with human fluency, substantially higher than all other automatic metrics.",
       "supported": "moderate"
     },
     {
       "claim": "Translation direction significantly affects code-switching quality, with Indic-to-English substantially outperforming English-to-Indic.",
       "evidence": "Table 9: mean difference 0.4684 for Accuracy and 0.4037 for Fluency (both p<.001). ANOVA Direction factor has highest F-scores: F=323.13 for Accuracy, F=293.31 for Fluency (Table 6).",
       "supported": "strong"
     },
     {
       "claim": "Llama3.1 8B consistently achieves higher accuracy and fluency than other models for code-switching generation.",
       "evidence": "Tables 3, 7: Llama3.1 significantly outperforms Aya23 and Llama3 for English-to-Indic accuracy (p<.001). For Indic-to-English, both Llama3 and Llama3.1 outperform Aya23.",
       "supported": "moderate"
     },
     {
       "claim": "EZSWITCH with LLM-generated translations performs comparably to Human ECT with gold translations.",
       "evidence": "Tukey's tests (Table 8) show no significant difference between Human ECT and EZSWITCH for either accuracy or fluency in both translation directions (p>0.69 for all comparisons).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Low inter-annotator agreement",
       "detail": "Krippendorff's alpha for Tamil Fluency is 0.321 and Accuracy is 0.445 (Table 11). Malayalam is similarly low (0.405/0.423). Only Hindi shows acceptable agreement (0.646/0.720). This undermines the reliability of the primary evaluation metric — human judgments — for two of three language pairs."
     },
     {
       "flag": "Very small effect sizes on a coarse scale",
       "detail": "Human ratings use a 1-3 discrete scale. Typical differences between methods are 0.05-0.19 points, meaning most improvements represent less than 10% of the scale range. The coarse granularity may be insufficient to capture meaningful quality differences."
     },
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any model (Llama3, Llama3.1, Aya23, GPT-4o-mini). These settings significantly affect generation quality and reproducibility."
     },
     {
       "flag": "Potential conflict of interest not disclosed",
       "detail": "Co-author Chaitanya Agarwal is affiliated with Deccan AI, which also provided the human annotation services used for the primary evaluation. This financial relationship is not discussed as a potential conflict."
     },
     {
       "flag": "No contamination analysis",
       "detail": "HinGE (2021) and Samanantar (2022) are public datasets that likely appear in the training data of Llama3/Llama3.1 (2024). If models memorized these sentences, generation quality would be artificially inflated. This is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Are multilingual models effective in code-switching?",
       "authors": ["Genta Indra Winata", "Samuel Cahyawijaya", "Zihan Liu", "Zhaojiang Lin", "Andrea Madotto", "Pascale Fung"],
       "year": 2021,
       "relevance": "Evaluates LLM multilingual capabilities in code-switching contexts, directly relevant to understanding model limitations."
     },
     {
       "title": "Multilingual large language models are not (yet) code-switchers",
       "authors": ["Ruochen Zhang", "Samuel Cahyawijaya", "Jan Christian Blaise Cruz", "Genta Winata", "Alham Aji"],
       "year": 2023,
       "relevance": "Demonstrates current LLM limitations in code-switching generation, establishing the problem this paper addresses."
     },
     {
       "title": "The decades progress on code-switching research in NLP: A systematic survey on trends and challenges",
       "authors": ["Genta Winata", "Alham Fikri Aji", "Zheng Xin Yong", "Thamar Solorio"],
       "year": 2023,
       "relevance": "Comprehensive survey of computational code-switching research, relevant as a survey methodology example."
     },
     {
       "title": "Prompting multilingual large language models to generate code-mixed texts: The case of south east asian languages",
       "authors": ["Zheng Xin Yong", "Ruochen Zhang", "Jessica Forde"],
       "year": 2023,
       "relevance": "Explores LLM prompting for code-switching generation, a direct comparison method for evaluating prompt-based approaches."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Technical report for the primary model (Llama3 8B) used in the evaluation experiments."
     },
     {
       "title": "Aya 23: Open weight releases to further multilingual progress",
       "authors": ["Viraat Aryabumi"],
       "year": 2024,
       "arxiv_id": "2405.15032",
       "relevance": "Technical report for Aya23, one of three models evaluated for multilingual code-switching generation."
     },
     {
       "title": "COMET: A neural framework for MT evaluation",
       "authors": ["Ricardo Rei", "Craig Stewart", "Ana C Farinha", "Alon Lavie"],
       "year": 2020,
       "relevance": "Machine translation evaluation metric used in this study; correlation analysis reveals its limitations for code-switching evaluation."
     },
     {
       "title": "Code-mixed probes show how pre-trained models generalise on code-switched text",
       "authors": ["Frances Adriana Laureano De Leon", "Harish Tayyar Madabushi", "Mark Lee"],
       "year": 2024,
       "relevance": "Investigates how pre-trained models handle code-switched text, relevant to understanding LLM code-switching capabilities."
     }
   ]
 }