ai-research-survey

scan.json (29279B)

---

 {
   "paper": {
     "title": "Retrieval-augmented generation in multilingual settings",
     "authors": [
       "Nadezhda Chirkova",
       "David Rau",
       "Hervé Déjean",
       "Thibault Formal",
       "Stéphane Clinchant",
       "Vassilina Nikoulina"
     ],
     "year": 2024,
     "venue": "KNOWLLM",
     "arxiv_id": "2407.01463",
     "doi": "10.48550/arXiv.2407.01463"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RAG improves performance across all 13 languages tested, but a gap remains between English and non-English. Translating system prompts into user languages and explicitly instructing the model to respond in the user language is critical for achieving high correct language rates. Multilingual-by-design generators (Command-R-35B) consistently outperform English-centric models for non-English, while BGE-m3 enables reliable cross-lingual retrieval. Code-switching in non-Latin alphabet languages and transliteration variability remain key limitations requiring evaluation metric adjustments.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code released at https://github.com/naver/bergen with multilingual-specific documentation at https://github.com/naver/bergen/blob/main/documentations/multilingual.md, stated in abstract and footnote 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available: MKQA, XOR-TyDi QA, TyDi QA, and Wikipedia (with HuggingFace links provided in footnotes 10 and 11). No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions int4 quantization and model names with HuggingFace links, but provides no requirements.txt, Dockerfile, or detailed environment/dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases code at the BERGEN library with dedicated multilingual documentation. Section 4 describes experimental details (retrieval setup, generation parameters, evaluation procedure) sufficient for reproduction via the released code."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (1, 5, 6, 7) report point estimates only (e.g., '58.4', '70.2'). No confidence intervals, error bars, or ± notation anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims of performance differences between models, prompts, and retrieval options are made throughout (e.g., 'Command-R-35B is the only model which consistently achieves high CLR') but no statistical significance tests are applied."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Tables show raw performance numbers for different conditions (e.g., no retrieval vs. retrieval in Table 1), but the paper never explicitly computes or reports effect sizes, relative improvements, or Cohen's d. The reader must compute differences from the tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "They use 2.7K MKQA samples and 3K XOR-TyDi QA validation examples without justifying these sample sizes. The subset selection from MKQA is explained (overlap with KILT NQ) but its adequacy for statistical conclusions is not discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single-run results with greedy decoding throughout. No variance, standard deviation, or spread measures reported. Section 4: 'We use greedy decoding' — one deterministic run per configuration."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: no-retrieval baseline (Table 1), different retrieval sources (English, user language, multilingual), default vs. advanced prompts (Table 5), English-centric vs. multilingual generators (Table 6), SPLADE+QT vs. BGE-m3 (Table 7), and oracle retrieval."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All models are contemporary (2023-2024): Command-R-35B, Mixtral-8x7B, SOLAR-10.7B, LLaMA-2-7B-chat, BGE-m3. These represent state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Systematic ablation across all pipeline components: retrieval language (Table 1), prompt formulation (Table 5), generator model (Table 6), and retrieval system (Table 7). Each table isolates one variable while controlling others."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics used: character 3-gram recall (task accuracy) and correct language rate (CLR, measuring whether the model responds in the user language). Both reported throughout Tables 5 and 6."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual inspection of 50 predictions each for Russian, Chinese, and French (Table 8). Authors categorize errors into system performance characteristics and data characteristics, identifying specific failure types like wrong retrieval, code-switching, and transliteration issues."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses standard held-out datasets: MKQA (derived from NQ), XOR-TyDi QA validation split, and TyDi QA. These are established evaluation sets separate from any model training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-language breakdowns across 13 languages in Table 1, per-prompt and per-model breakdowns in Tables 5-6, and per-error-type breakdown in Table 8. Results are never reported as only aggregates."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 provides detailed manual error analysis: wrong retrieval, wrong response with correct retrieval, code-switching, transliteration issues, fluency errors, and data quality problems. Section 5 discusses these systematically."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative findings reported: English-centric models frequently reply in English for non-English queries (Table 5), Mixtral only works well for its pretrained languages (Table 6), Command-R's English recall is lower than SOLAR's due to 'curse of multilinguality' (Section 5), and code-switching remains unsolved."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: 'task-specific prompt engineering is needed' (Table 5), 'evaluation metrics need adjustments' (Table 3 and Section 3 evaluation discussion), 'frequent code-switching in non-Latin alphabet languages' (Table 8), all directly supported by experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Claims are primarily comparative ('translating prompts improves CLR', 'retrieval from multilingual Wikipedia is beneficial'). The ablation design with controlled single-variable manipulation across Tables 5-7 provides adequate evidence for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are scoped to the tested setting: 13 specific languages, MKQA and XOR-TyDi QA datasets, zero-shot setting, open QA task with Wikipedia. The Limitations section explicitly states these boundaries and calls for extending to other tasks and domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers brief mentions (SOLAR's multilingual ability 'attributed to accidental multilingual data in pretraining', Command-R's lower English scores due to 'curse of multilinguality') but these are passing observations, not substantive discussion of alternative explanations for the main findings."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures character 3-gram recall and correct language rate, and frames claims at the same granularity as these measurements. No broader framing beyond what the metrics measure (e.g., they don't claim to measure 'RAG quality' generally, just QA accuracy and language correctness on specific benchmarks)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers provided: 'LLaMA-2-7B-chat', 'SOLAR-10.7B', 'Mixtral-8x7B', 'Command-R-35B' with HuggingFace links for Command-R (CohereForAI/c4ai-command-r-v01), BGE-m3, and NLLB-600M. Sufficient to identify exact model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides the full text of all six system prompt variants used. The {UL} placeholder is clearly defined as the user language name, and all test languages are enumerated, so prompts can be fully reconstructed."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 reports: greedy decoding, maximum 128 new tokens, int4 quantization, top-50 retrieval then top-5 reranking, 100-word passage chunks (100 Unicode characters for CJK/Thai)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The pipeline is a straightforward retrieve-then-generate architecture without agents, tool use, or iterative reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents preprocessing: Wikipedia split into 100-word chunks with article title prepended (100 Unicode characters for non-whitespace-separated languages), MKQA subset selected via overlap with KILT NQ, evaluation preprocessing includes lower-casing, punctuation removal, and article removal."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated 'Limitations' section after Section 6, discussing scope restrictions (QA task, Wikipedia datastore), excluded components (query reformulation, context post-processing), and single retriever choice."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: only one retriever/reranker model used (BGE-m3), BGE-m3 is in-domain since it was trained on Wikipedia data, evaluation limited to QA task, prompt translations may have quality issues (footnote 7), and evaluation data has ambiguous/incorrect labels (Table 8)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section explicitly states what was NOT tested: other tasks beyond QA, other domains beyond Wikipedia, query reformulation, context post-processing (filtering irrelevant passages). Also Section 6 lists specific future directions as implicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation data is from publicly available datasets with provided links: MKQA, XOR-TyDi QA, TyDi QA, Wikipedia (via HuggingFace). Code is released for reproducing results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Dataset sources described: MKQA from NQ (10K examples translated to 25 languages, 2.7K subset selected), XOR-TyDi QA (40K questions in 7 languages, 3K validation), Wikipedia passage construction documented. Table 4 provides detailed statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (MKQA, XOR-TyDi QA, Wikipedia)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented in Section 4: Wikipedia articles split into 100-word passages with title prepended, MKQA subset filtered via KILT NQ overlap (recovering relevant document information), retrieval of top-50 passages then reranking to top-5, evaluation with preprocessed character 3-gram matching."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source disclosed. The Acknowledgments section thanks named individuals but mentions no grants, funding agencies, or corporate sponsors. All authors are from NAVER LABS Europe (corporate research lab)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with NAVER LABS Europe in the paper header. They evaluate third-party models (Command-R by Cohere, BGE-m3 by BAAI, LLaMA by Meta, etc.) rather than NAVER products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is explicitly disclosed. As NAVER LABS employees, their research is implicitly funded by NAVER, a tech company with search products that could benefit from RAG advances. Independence cannot be assessed without disclosure."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, no patent disclosures, and no financial interest declarations anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the four generator models (LLaMA-2, SOLAR, Mixtral, Command-R) or the retriever (BGE-m3). Languages in pretraining are mentioned for some models, but not temporal cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MKQA or XOR-TyDi QA examples appeared in the training data of any evaluated model. MKQA derives from Natural Questions (2019), and all models were trained after this date."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MKQA is derived from Natural Questions (published 2019) and uses Wikipedia, both of which are highly likely in the training data of all evaluated models. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Manual error analysis in Table 8 is performed by the researchers, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates models on public benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-query cost reported despite running 4 generators across 13 languages on thousands of examples with retrieval and reranking."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total computational budget stated. Only int4 quantization is mentioned without stating what hardware was used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy decoding makes generation deterministic, but this is not explicitly discussed as a design choice for reproducibility. No seed sensitivity analysis is reported for any component."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. Results appear to be from single runs with greedy decoding, but this is not confirmed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests multiple prompt formulations and configurations but does not describe these as a hyperparameter search and does not report the total configurations tried or compute spent on exploration."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 ('Best performing configuration to be used as a strong baseline') explicitly identifies the best configuration (Command-R-35B, BGE-m3, 'Reply short in UL (UL)' prompt, multilingual Wikipedia retrieval) and justifies each choice based on the systematic comparisons in Tables 1, 5-7."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all. Since no p-values are computed, multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors build and evaluate their own mRAG pipeline but do not acknowledge the bias of evaluating their own system or discuss how this might affect their assessment of component contributions."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 7B (LLaMA-2) to ~46B parameters (Mixtral-8x7B) and 35B (Command-R), but performance is never reported as a function of compute budget. The compute cost difference is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "MKQA and XOR-TyDi QA are used without formal discussion of whether they validly measure multilingual RAG capability. Table 8 identifies data quality issues (ambiguous questions, wrong labels) but does not frame this as a construct validity concern."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The retrieve-then-generate pipeline has no scaffolding confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. MKQA derives from Natural Questions (2019) and Wikipedia, both created before all evaluated models' training data. Models may have seen both the questions and Wikipedia passages during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The no-retrieval baseline implicitly addresses whether models already know answers, but this is not framed as feature leakage analysis."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. MKQA and Wikipedia are both likely in the training data of the evaluated models, creating potential non-independence between training and evaluation data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method employed. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipeline."
       }
     }
   },
   "claims": [
     {
       "claim": "RAG brings substantial performance improvement across all 13 languages tested.",
       "evidence": "Table 1 shows consistent improvement from no-retrieval to retrieval conditions across all languages on both MKQA and XOR-TyDi QA (e.g., English MKQA: 58.4 → 70.2, Arabic: 26.4 → 49.0).",
       "supported": "strong"
     },
     {
       "claim": "Translating system prompts into user languages and explicitly specifying target language are critical for achieving high correct language rate.",
       "evidence": "Table 5 shows CLR improvements across all settings. For Command-R with English retrieval, Korean CLR goes from 54.3% (default) to 100% (translated prompt with UL instruction).",
       "supported": "strong"
     },
     {
       "claim": "Multilingual-by-design generator models are essential for consistent generation across languages.",
       "evidence": "Table 6 shows Command-R-35B achieves CLR >98% across all tested languages with appropriate prompts, while English-centric LLaMA-2-7B achieves only 4.3-62.8% CLR and often replies in English.",
       "supported": "strong"
     },
     {
       "claim": "BGE-m3 enables reliable cross-lingual retrieval without requiring query translation.",
       "evidence": "Table 7 shows BGE-m3 outperforms SPLADE+QT (query translation) on both retrieval recall@5 and downstream character 3-gram recall across all languages.",
       "supported": "strong"
     },
     {
       "claim": "Retrieving from multilingual Wikipedia is beneficial compared to monolingual retrieval in most cases.",
       "evidence": "Table 1 'All langs' column often matches or exceeds both English-only and user-language-only retrieval (e.g., German MKQA: 66.9 vs 64.6/65.5). However, patterns vary by dataset: MKQA favors English, XOR-TyDi QA favors user language.",
       "supported": "moderate"
     },
     {
       "claim": "Character 3-gram recall is more robust than word-level matching for multilingual evaluation.",
       "evidence": "Table 3 illustrates that word-level exact match gives 0% for 'Sofia Kovalevskaia' vs ground truth 'Sofya Kovalevskaya', while character 3-gram recall gives 69.2%. However, this is demonstrated by example rather than systematic comparison.",
       "supported": "moderate"
     },
     {
       "claim": "Code-switching in non-Latin alphabet languages is a persistent limitation of current multilingual LLMs.",
       "evidence": "Table 8 manual inspection: 5 out of 50 Russian predictions and 6 out of 50 Chinese predictions had named entities in English. Even with explicit NE-in-UL prompting, the issue is not fully resolved (Table 5).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical testing",
       "detail": "All performance comparisons across models, prompts, languages, and retrieval options are based solely on point-estimate comparisons with no confidence intervals, significance tests, or uncertainty quantification. With thousands of examples per condition, even small differences could be tested."
     },
     {
       "flag": "Contamination not addressed",
       "detail": "MKQA derives from Natural Questions (2019) and Wikipedia, both very likely in the training data of all evaluated models (2023-2024 vintage). The no-retrieval baseline may partially reflect memorized answers rather than true zero-knowledge performance, but this is never discussed."
     },
     {
       "flag": "Small manual evaluation sample",
       "detail": "Error analysis (Table 8) is based on only 50 examples per language for 3 of 13 languages. This is too small for reliable error rate estimates and may not represent the full distribution of error types."
     },
     {
       "flag": "Single retriever evaluated",
       "detail": "Only BGE-m3 is used as the multilingual retriever/reranker. The comparison with SPLADE (Table 7) is English-only. The paper acknowledges this in Limitations but the single-system evaluation limits generalizability of retrieval-related findings."
     },
     {
       "flag": "No compute or cost reporting",
       "detail": "Running 4 generators × 6 prompts × multiple retrieval settings × 13 languages on thousands of examples represents substantial compute, but no GPU hours, hardware, latency, or cost figures are provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper that introduced the retrieve-then-generate paradigm widely used in LLM applications."
     },
     {
       "title": "Self-RAG: Learning to retrieve, generate, and critique through self-reflection",
       "authors": ["Akari Asai", "Zeqiu Wu", "Yizhong Wang"],
       "year": 2024,
       "relevance": "Proposes self-reflective RAG where the model decides when retrieval is needed, relevant to RAG methodology and evaluation."
     },
     {
       "title": "BGE m3-embedding: Multi-lingual, multi-functionality, multi-granularity text embeddings through self-knowledge distillation",
       "authors": ["Jianlv Chen", "Shitao Xiao", "Peitian Zhang"],
       "year": 2024,
       "relevance": "The multilingual embedding model used as the primary retriever/reranker, core to multilingual retrieval capability evaluation."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "relevance": "Major open-source LLM evaluated as an English-centric baseline for multilingual generation capability."
     },
     {
       "title": "Mixtral of experts",
       "authors": ["Albert Q. Jiang", "Alexandre Sablayrolles"],
       "year": 2024,
       "relevance": "Mixture-of-experts LLM evaluated for multilingual generation, representing a multilingual-by-design architecture."
     },
     {
       "title": "Bergen: A benchmarking library for retrieval-augmented generation",
       "authors": ["David Rau", "Hervé Déjean", "Nadezhda Chirkova"],
       "year": 2024,
       "relevance": "The RAG benchmarking framework on which this multilingual evaluation is built, relevant to RAG evaluation methodology."
     },
     {
       "title": "One question answering model for many languages with cross-lingual dense passage retrieval",
       "authors": ["Akari Asai", "Xinyan Yu", "Jungo Kasai"],
       "year": 2021,
       "relevance": "Introduced the CORA approach and XOR-TyDi QA for multilingual open QA, the primary prior work and evaluation benchmark."
     },
     {
       "title": "Making retrieval-augmented language models robust to irrelevant context",
       "authors": ["Ori Yoran", "Tomer Wolfson", "Ori Ram"],
       "year": 2024,
       "relevance": "Addresses RAG robustness to irrelevant context, relevant to understanding RAG pipeline reliability."
     },
     {
       "title": "MKQA: A linguistically diverse benchmark for multilingual open domain question answering",
       "authors": ["Shayne Longpre", "Yi Lu", "Joachim Daiber"],
       "year": 2021,
       "relevance": "One of two primary evaluation benchmarks used, providing parallel multilingual QA evaluation across 25 languages."
     },
     {
       "title": "No language left behind: Scaling human-centered machine translation",
       "authors": ["NLLB Team", "Marta R. Costa-jussà"],
       "year": 2022,
       "relevance": "The NLLB translation model used for query translation baseline in cross-lingual retrieval experiments."
     }
   ]
 }