ai-research-survey

scan.json (26144B)

---

 {
   "paper": {
     "title": "Lost in the Mix: Evaluating LLM Understanding of Code-Switched Text",
     "authors": ["Amr Mohamed", "Yang Zhang", "Michalis Vazirgiannis", "Guokan Shang"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.14012"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository URL is provided: https://github.com/amr-mohamedd/Lost-in-the-Mix.git (Section 1, footnote 1). The paper states 'Code and data are publicly available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code and data are publicly available' (Section 1) and uses publicly available benchmarks (Belebele, MMLU, XNLI). The generated code-switched variants are described as available via the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states 'All experiments were conducted using NVIDIA A100 (40GB VRAM) and A10 (24GB VRAM) GPU clusters.' Appendix D specifies training hyperparameters including mixed-precision BF16, 4096-token window, batch size of four. However, no requirements.txt or Dockerfile is mentioned; the hardware and key training parameters are specified but full software dependencies are not documented."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a GitHub link is provided, the paper itself does not include step-by-step reproduction instructions, a 'Reproducing Results' section, or specific commands to run experiments. The methodology is described but operational reproduction steps are absent from the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., accuracy values like 0.66, 0.70) without confidence intervals, error bars, or any uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'performance declines,' 'improved comprehension') but no statistical significance tests (p-values, t-tests, etc.) are reported anywhere. Differences are stated as raw numerical comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports accuracy deltas (Delta Acc) throughout, showing both baseline and code-switched values (e.g., 'Llama 3B decreased from 0.54 to 0.43 on EN→DE (Delta = -0.11)'). This provides sufficient context to understand the magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the sample sizes used. The 300-sample test sets for CSW approach evaluation (Section 3.4) and 100-sample comparisons (Appendix B) are stated without power analysis or rationale for why these sizes are adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experiment. Results appear to be single-run evaluations with no information about variability across runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Monolingual English performance serves as the baseline throughout all experiments (Tables 1-5), allowing direct comparison of code-switched performance against the original benchmark performance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models evaluated include contemporary LLMs: LLaMA 3.1/3.2 (2024), Qwen 2.5 (2025), Mistral 7B v0.3 (2023), and ALLaM 7B (2024). The benchmarks (Belebele, MMLU, XNLI) are standard and widely used."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 presents ablation studies: Section 6.1 reverses language roles (English as embedded), Section 6.2 tests extreme multi-language CSW with different language combinations (Settings 1-3). Section 7 ablates mitigation strategies (prompting vs. fine-tuning)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are formally defined in Section 3.5: accuracy, weighted average accuracy, and accuracy delta (Delta Acc). Results are also broken down by individual benchmark (Belebele, MMLU, XNLI) in Figure 2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 describes bilingual annotators conducting pairwise preference evaluations of CSW text quality between Claude and GPT-4o outputs (100 examples per language pair, Table 6), assessing which 'sounded more natural.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses established benchmark test sets (Belebele, MMLU, XNLI) that are separate from any training data. For the fine-tuning experiment (Section 7.2), training data comes from TED Talk transcripts while evaluation is on the benchmark test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (8 models), per embedded language (4 languages), and per benchmark task (Belebele, MMLU, XNLI) as shown in Figure 2 and Tables 1-5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 discusses cases where prompting was detrimental: 'Llama 70B, especially on EN→AR and EN→ZH, where performance fell by 13 and 17 points respectively.' Section 8 discusses where models 'struggle to process disrupted monolingual structures.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: prompt-based mitigation was 'detrimental' for Llama and Mistral models (Section 7.1, Table 5), fine-tuning achieved only 'partial recovery' (Section 7.2), and H1 (degradation proportional to linguistic distance) was not fully supported by the data."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (degradation when foreign tokens disrupt English, improvement when English embedded in other languages, mixed prompting results, fine-tuning as more stable path) are all supported by corresponding results in Tables 1-5 and Figure 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'embedding English tokens into other languages often improves comprehension' and 'code-switching introduces specific comprehension challenges.' These are based on pre-post comparisons without controlling for confounds like tokenization differences, token frequency effects, or benchmark translation quality. No causal identification strategy is used."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract frame results broadly as 'LLM Understanding of Code-Switched Text,' but the study tests only 8 models, 5 languages, 3 benchmarks, and 2 CSW methods (noun-token and ratio-token). The Limitations section acknowledges only CSW method constraints, not the narrow language/model scope relative to the broad framing."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations. The claim that degradation stems from 'structural vulnerability' rather than 'token-level unfamiliarity' (Section 8) is asserted without ruling out confounds like tokenization artifacts, training data distribution, or benchmark translation quality affecting results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Model families and sizes are specified (e.g., 'LLaMA 3.2 Instruct (3B)', 'Qwen 2.5 Instruct (3B, 7B, 72B)', 'Mistral 7B Instruct (v0.3)'), but exact model version strings or snapshot dates are not provided. For the generation models, only 'Claude 3.5 Sonnet' and 'GPT-4o' are given without version identifiers or API snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in the appendices: CSW generation prompts (Figures 4-6), evaluation prompt (Figure 7), mitigation prompts for each benchmark (Figures 8-10), and instruction tuning templates (Figures 11-15)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D reports fine-tuning hyperparameters: learning rate 2e-6, linear decay, 5% warmup, BF16, 4096-token window, batch size 4, single epoch. However, inference hyperparameters (temperature, top-p) for the evaluated models are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-step CSW generation pipeline is described in detail in Section 3.3: step 1 identifies placeholder positions, step 2 fills placeholders with target language words. The alignment-based pipeline components (AWESOME aligner, LaBSE embeddings, Stanza POS tagger) are specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data preprocessing is documented: Section 3.3-3.4 describes the CSW generation pipeline. Section 7.2 documents the fine-tuning data preparation: selecting TED Talk sentences >70 words, yielding ~3,650 pairs per language, totaling ~14,600 training samples. The EleutherAI harness adaptation is mentioned in Section 4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after Section 8 (Discussion and Conclusion), discussing constraints of the noun-token approach and the fixed 20% substitution rate."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section is brief and mostly generic: it mentions that 'more complex forms of code-switching may induce more severe performance degradation' and that the 20% ratio was fixed. These are methodological scope notes, not specific threats to validity of the reported findings (e.g., no discussion of CSW generation quality affecting results, possible benchmark translation artifacts, or tokenization confounds)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section mentions future work directions but does not explicitly state what the results do NOT show. There are no statements bounding the claims to the specific models, languages, or benchmarks tested. The paper does not say, e.g., 'our results do not generalize to other language families or proprietary models.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code and data are publicly available' with a GitHub link (Section 1). The underlying benchmarks (Belebele, MMLU, XNLI) are publicly available standard datasets."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection/generation process is described in detail across Sections 3.2-3.4 and Appendices B-D: how CSW variants were generated, which approaches were compared, and how the fine-tuning dataset was constructed from TED Talk transcripts."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B.1 mentions 'bilingual annotators' who conducted preference evaluations, but provides no information about how they were recruited, their qualifications, number of annotators, or potential selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from benchmark selection to CSW generation to evaluation is documented: benchmark selection (Section 4), CSW generation methods and selection (Section 3.3-3.4, Appendix B), evaluation setup using EleutherAI harness (Section 4), and fine-tuning data preparation (Section 7.2, Appendix D)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. The authors are affiliated with MBZUAI and Ecole Polytechnique, but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the first page: 1MBZUAI, 2Ecole Polytechnique."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so it is impossible to assess whether any funder has a stake in the outcome. The absence of any funding disclosure means this criterion cannot be satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is provided anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple LLMs on established benchmarks (MMLU, Belebele, XNLI) but does not state the training data cutoff dates for any of the models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. MMLU (published 2020) and XNLI (published 2018) are well-known benchmarks that are very likely present in training data of models trained after their publication. This contamination risk is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU (2020), XNLI (2018), and Belebele (2023) were all published before the training cutoffs of the models tested (LLaMA 3, Qwen 2.5, etc.). One could argue that the code-switched variants are novel, but the original English benchmarks are likely contaminated, and the paper does not discuss how this might affect the baseline comparisons or the measured accuracy deltas."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluation was limited to pairwise preference judgments of CSW text quality by bilingual annotators (Appendix B.1), not a human subjects study requiring pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The paper involves annotator preference judgments, not a human subjects study with participants as research subjects. IRB approval is not typically required for this type of evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the sense of a human subjects study. The bilingual annotators are evaluators of text quality, not research subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects. The annotators are part of the methodology, not the subject population."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects experiment. No randomization of participants to conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects experiment requiring blinding of participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study with participant attrition."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example costs are reported, despite using commercial APIs (Claude 3.5 Sonnet, GPT-4o) extensively for CSW generation and evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states: 'The compute allocation totaled 22 GPU-days, comprising 8 GPU-days on 8xA100 nodes and 14 GPU-days on 4xA10 nodes.'"
       }
     }
   },
   "claims": [
     {
       "claim": "Embedding non-English tokens into English text consistently degrades LLM performance, even when switches follow linguistic constraints.",
       "evidence": "Tables 1 and 2 show consistent drops across all 8 models and 4 embedded languages for both noun-token and ratio-token CSW. E.g., LLaMA-70B drops from 0.70 to 0.66-0.67 (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Embedding English tokens into non-English matrix languages often improves comprehension.",
       "evidence": "Table 3 (Section 6.1) shows improvements in most model-language combinations, e.g., Mistral 7B Arabic rising from 0.35 to 0.48 (Delta = +0.13). Only one case showed a minor drop (Qwen 7B on Chinese, Delta ~ -0.01).",
       "supported": "strong"
     },
     {
       "claim": "Code-switching complexity does not linearly correlate with performance degradation.",
       "evidence": "Table 4 (Section 6.2) shows that extreme CSW with 4 embedded languages (Setting 3) does not consistently yield the worst results across all models. Some models perform better with more languages than fewer.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt-based mitigation yields inconsistent results across models.",
       "evidence": "Table 5 (Section 7.1) shows Qwen family models improve with prompting while Llama and Mistral models degrade significantly (e.g., Llama 70B drops by 13-17 points on some language pairs).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning on code-switched data leads to more stable performance recovery than prompting.",
       "evidence": "Figure 3 and Section 7.2 show CSW-Llama 8B recovers +0.03 to +0.04 points across language pairs after fine-tuning. However, this is tested on only one model (Llama 8B) and recovery is partial.",
       "supported": "weak"
     },
     {
       "claim": "Degradation does not stem solely from token-level unfamiliarity but from a more fundamental processing difficulty.",
       "evidence": "Section 8 argues that similar degradation under both linguistically constrained (noun-token) and random (ratio-token) CSW suggests a structural vulnerability. However, this interpretation is not tested against alternative explanations like tokenization artifacts.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper systematically evaluates 8 LLMs on code-switched versions of Belebele, MMLU, and XNLI benchmarks across 5 languages. Embedding non-English tokens into English consistently degrades performance (up to -0.11 accuracy), while embedding English into other languages often improves performance (up to +0.13), revealing an asymmetric linguistic vulnerability. Prompt-based mitigation helps some model families (Qwen) but harms others (Llama, Mistral), while fine-tuning on CSW data provides more consistent but partial recovery, tested only on Llama 8B.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims are based on raw accuracy differences without any significance testing. With single-run evaluations and no variance reporting, it is impossible to know whether observed differences (e.g., 1-2 percentage point changes) are meaningful or within noise."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "MMLU (2020) and XNLI (2018) are well-known benchmarks almost certainly present in the training data of the models tested (LLaMA 3, Qwen 2.5, etc.). The baseline English accuracy may reflect memorization rather than comprehension, which would confound the interpretation of accuracy deltas under code-switching."
     },
     {
       "flag": "Fine-tuning claim based on single model",
       "detail": "The claim that fine-tuning is a 'more stable path to degradation mitigation' is based on experiments with only Llama 8B. Generalizing from one model to a broad conclusion about mitigation strategies is unsupported."
     },
     {
       "flag": "No inference temperature or sampling parameters reported",
       "detail": "While fine-tuning hyperparameters are specified, the inference settings (temperature, top-p, etc.) for all 8 evaluated models are not reported. These settings significantly affect LLM outputs and reproducibility."
     },
     {
       "flag": "Annotator details missing",
       "detail": "The human evaluation (bilingual annotators, Appendix B.1) provides no information about the number of annotators, their qualifications, inter-annotator agreement, or recruitment process."
     }
   ],
   "cited_papers": [
     {
       "title": "Multilingual large language models are not (yet) code-switchers",
       "authors": ["Ruochen Zhang", "Samuel Cahyawijaya", "Jan Christian Blaise Cruz", "Genta Winata", "Alham Fikri Aji"],
       "year": 2023,
       "relevance": "Directly relevant study evaluating LLM code-switching capabilities, establishing the baseline that this paper extends."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-bench and chatbot arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Foundational paper for the LLM-as-a-judge evaluation methodology used in this paper for CSW quality assessment."
     },
     {
       "title": "The llama 3 herd of models",
       "authors": ["Aaron Grattafiori", "Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Model paper for LLaMA 3 family used as primary evaluation targets in the study."
     },
     {
       "title": "Qwen3 technical report",
       "authors": ["An Yang", "Anfeng Li", "Baosong Yang"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Model paper for Qwen family used as evaluation targets; Qwen showed distinctive behavior under prompt-based mitigation."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2020,
       "arxiv_id": "2009.03300",
       "relevance": "MMLU benchmark paper; one of the three core benchmarks used for evaluating code-switching impact on LLM comprehension."
     },
     {
       "title": "A survey of large language models",
       "authors": ["Wayne Xin Zhao", "Kun Zhou", "Junyi Li"],
       "year": 2023,
       "arxiv_id": "2303.18223",
       "relevance": "Comprehensive LLM survey providing context for the capabilities and limitations of models evaluated in this study."
     },
     {
       "title": "The language model evaluation harness",
       "authors": ["Leo Gao", "Jonathan Tow", "Baber Abbasi"],
       "year": 2024,
       "relevance": "Evaluation infrastructure (EleutherAI harness) adapted for the code-switched benchmark evaluations in this study."
     },
     {
       "title": "Linguistics theory meets LLM: Code-switched text generation via equivalence constrained large language models",
       "authors": ["Garry Kuwanto", "Chaitanya Agarwal", "Genta Indra Winata", "Derry Tanti Wijaya"],
       "year": 2024,
       "arxiv_id": "2410.22660",
       "relevance": "Prior work on linguistically constrained LLM-based code-switched text generation that this paper builds upon."
     },
     {
       "title": "LLM-based code-switched text generation for grammatical error correction",
       "authors": ["Tom Potter", "Zheng Yuan"],
       "year": 2024,
       "relevance": "Prior work on LLM-based CSW generation approaches that motivated the generation methodology in this study."
     },
     {
       "title": "Beyond metrics: evaluating LLMs' effectiveness in culturally nuanced, low-resource real-world scenarios",
       "authors": ["Millicent Ochieng", "Varun Gumma", "Sunayana Sitaram"],
       "year": 2024,
       "arxiv_id": "2406.00343",
       "relevance": "Related work on evaluating LLM performance in multilingual and culturally diverse settings."
     },
     {
       "title": "Allam: Large language models for arabic and english",
       "authors": ["M Saiful Bari", "Yazeed Alnumay"],
       "year": 2024,
       "arxiv_id": "2407.15390",
       "relevance": "Model paper for ALLaM, the Arabic-specialized LLM used as one of the evaluation targets in this study."
     }
   ]
 }