ai-research-survey

scan.json (25886B)

---

 {
   "paper": {
     "title": "Empowering Low-Resource Languages: TraSe Architecture for Enhanced Retrieval-Augmented Generation in Bangla",
     "authors": [
       "Atia Shahnaz Ipa",
       "Mohammad Abu Tareq Rony",
       "Mohammad Shariful Islam"
     ],
     "year": 2025,
     "venue": "LM4UC 2025 (Workshop on Language Models for Underserved Communities, ACL 2025)",
     "doi": "10.18653/v1/2025.lm4uc-1.2"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The TraSe architecture combines translative prompting (translate Bangla query to English, generate answer, translate back) with a selection mechanism that picks the best answer from two prompting strategies. On a 200-question Bangla QA dataset from Wikipedia, TraSe achieves 34% accuracy with automatic retrieval (BanglaBERT embeddings) and 63% with human-in-the-loop retrieval, outperforming zero-shot (22%/51%), 2-shot, Self-Ask, and ReAct baselines. Translative prompting is particularly effective for text-based (vs. number-based) answers.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided in the abstract: https://github.com/Atia6/TraSe-Bangla-RAG."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states they created 200 QA pairs from Bangla Wikipedia dump but only explicitly mentions code availability. The dataset release is not explicitly confirmed; only 'The code is available' is stated."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Llama 2 7B, transformers library, LangChain, and bfloat16, but provides no requirements.txt, Dockerfile, or detailed environment setup with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README content or reproduction steps are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 3 and Figure 4 are reported as point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims TraSe 'outperforms' baselines and shows 'significant improvements' but provides no statistical significance tests. Comparisons are based solely on numerical differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports absolute accuracy and F1 scores for all methods with and without retrieval, providing baseline context (e.g., 22% for 0-shot direct vs. 33% for 0shot+Translative with BanglaBERT)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains only 200 QA pairs from 27 Wikipedia articles. No justification is given for why 200 questions or 27 articles were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multi-run results are reported. All results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against four baseline methods: zero-shot, 2-shot, Self-Ask, and ReAct (Section 3.2, Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include ReAct (Yao et al., 2023) and Self-Ask (Press et al., 2023), which are contemporary prompting methods for the task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows results for different TraSe combinations (0shot+Translative, 2shot+Translative, SelfAsk+Translative, ReAct+Translative) compared to standalone methods, effectively demonstrating which component combinations matter."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Both accuracy and F1 score are reported for all methods in Table 3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 states 'The generated answers were manually evaluated and assigned as right or wrong answers. Based on manual evaluation the accuracy has been determined.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No mention of train/dev/test splits. The temperature was tuned on what appears to be the same 200 questions used for final evaluation (Figure 1), with no held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 breaks down results by text-based vs. number-based answers, and Table 3 breaks down by retrieval method (BanglaBERT, Bert-multilingual, HIL)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides one example in Section 3.6 illustrating why exact match fails as a metric, but does not analyze where or why the system produces wrong answers for the 37-66% of cases it fails on."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "All TraSe variants are presented as improvements. The paper does not discuss any approaches that were tried and abandoned, or configurations that degraded performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 34% accuracy with automatic retrieval and 63% with HIL retrieval are confirmed by Table 3 (0shot+Translative: 0.34 with BanglaBERT, 0.63 with HIL). The claim of 'outperforming baseline methods' is supported by the comparison table."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims TraSe 'improves' and 'enhances' performance. The experimental design holds the model constant (Llama 2 7B) and varies only the prompting/selection method, which is adequate single-variable manipulation for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Enhanced Retrieval-Augmented Generation in Bangla' broadly, but the study uses only one model (Llama 2 7B), one dataset (200 questions from 27 Wikipedia articles), and one language. The abstract claims 'potential to enhance question-answering systems for Bangla and similar languages' without testing any other language."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered. For example, the translative method's advantage could be because Llama 2 has far more English training data, but this confound is not discussed. The selection mechanism's advantage from having two answer candidates is also not analyzed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on QA pairs and frames results as QA accuracy. The measurements match the granularity of the claims without overreaching into broader constructs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'Llama 2 7B model' but does not specify the exact variant (base vs. chat vs. instruct) or the HuggingFace model ID. BanglaBERT and bert-base-multilingual-cased are named but without version numbers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The prompting strategies (zero-shot, 2-shot, translative, etc.) are described conceptually and in flowcharts (Figures 2-3), but the actual prompt text used is never provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 reports temperature (0.0001), top_k (10), max sequence length (3000 tokens), and data type (bfloat16)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The TraSe architecture is described in Section 3.5 and Figure 3, showing the multi-step pipeline: embedding, retrieval, parallel prompting (translative + baseline), and LLM-based answer selection."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Table 1 document the preprocessing: Bangla Wikipedia dump → 27 articles → chunking into 5-sentence segments (710 chunks) → 200 QA pairs with 3 related contexts each."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, discussing the use of a single language model and smaller sample size."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies study-specific threats: single language model may not generalize across models, and the smaller sample size (200 questions) 'may affect the generalizability of the results.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section mentions needing more models and languages, but does not explicitly state what the results do NOT show (e.g., 'our results apply only to Bangla Wikipedia QA with Llama 2 7B')."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 200 QA pairs and their manual evaluation labels are not explicitly made available. Only code is mentioned as released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the data source (Bangla Wikipedia dump), structure (27 articles, 53,575 words, 710 chunks, 200 QA pairs), chunking strategy (5 sentences), and answer types (70 text-based, 130 number-based)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. Data comes from Bangla Wikipedia, a public data source."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from Wikipedia dump to final QA pairs has gaps: how were the 27 specific articles selected from the dump? How were the 200 questions created from those articles? Who authored the questions? These steps are unexplained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Khulna University of Engineering & Technology and Noakhali Science & Technology University, Bangladesh. No commercial product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of any funding statement means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama 2 7B to answer questions derived from Bangla Wikipedia but never states the model's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The QA pairs are derived from Bangla Wikipedia, which is almost certainly included in Llama 2's training data. This overlap is never discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Since the benchmark is derived from Wikipedia and Llama 2 was trained on internet data including Wikipedia, the model may already know the answers. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The 'Human-in-the-Loop' component refers to providing oracle retrieval contexts, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-query compute time is reported despite TraSe requiring multiple LLM calls per query."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No hardware details, GPU hours, or total compute budget are mentioned."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results reported. All results appear to be from a single run with a fixed temperature of 0.0001."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results appear to be from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Figure 1 shows 6 temperature values tested, but no systematic search method or total budget is described. Other hyperparameters (top_k=10) appear chosen without stated justification."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.0001 was selected based on accuracy in Figure 1, but this appears to be evaluated on the same data used for final results — no separate validation set is mentioned."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own TraSe system against baselines they implemented. No acknowledgment of potential author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "TraSe requires at least 3 LLM calls per query (translative + baseline + selector) vs. 1 for baselines, but this computational cost difference is never discussed or compared against the accuracy gain."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses a custom 200-question benchmark without discussing whether these 200 questions adequately measure RAG capability for Bangla, or whether the text/number split is representative."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The scaffold (TraSe architecture) IS the method being evaluated. All comparisons use the same underlying model (Llama 2 7B) with different prompting/scaffolding strategies as the independent variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Llama 2 was trained on data including Wikipedia (collected before July 2023). The Bangla Wikipedia dump used for the benchmark predates this. Temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the retrieval contexts or question phrasing leak answer information beyond what would be available in a real deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 200 questions from 27 articles introduce structural dependencies (e.g., multiple questions from the same article sharing context)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied despite the high risk of Wikipedia content being in Llama 2's training data."
       }
     }
   },
   "claims": [
     {
       "claim": "TraSe achieves 34% accuracy with automatic retrieval (BanglaBERT) and 63% with Human-in-the-Loop retrieval for Bangla QA.",
       "evidence": "Table 3 shows 0shot+Translative TraSe achieving F1=0.50/accuracy=0.33 with BanglaBERT, F1=0.51/accuracy=0.34 with BanglaBERT (2shot variant), and F1=0.77/accuracy=0.63 with HIL retrieval.",
       "supported": "moderate"
     },
     {
       "claim": "TraSe outperforms all baseline prompting methods across retrieval settings.",
       "evidence": "Table 3 shows TraSe variants consistently achieving higher accuracy and F1 scores than standalone methods (zero-shot: 22%→33% with BanglaBERT, 51%→63% with HIL). However, no statistical tests confirm these differences are significant on a 200-question dataset.",
       "supported": "weak"
     },
     {
       "claim": "Translative prompting is particularly effective for text-based answers compared to other prompting methods.",
       "evidence": "Figure 4 shows the Translative method achieving the highest accuracy for text-based answers across all retrieval settings (0.28 for BanglaBERT text vs. 0.18 for zero-shot, 0.61 for HIL text vs. 0.36 for zero-shot).",
       "supported": "moderate"
     },
     {
       "claim": "The TraSe architecture marks a significant advancement in RAG for low-resource languages.",
       "evidence": "Only demonstrated on Bangla with a single model (Llama 2 7B) and 200 questions. Peak accuracy is 63% with oracle retrieval and 34% with automatic retrieval. No other low-resource languages tested.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny dataset",
       "detail": "Only 200 QA pairs from 27 Wikipedia articles. This is extremely small for making generalizable claims about RAG performance for an entire language. With 70 text-based and 130 number-based questions, per-category samples are even smaller."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "The paper claims 'significant improvements' and that TraSe 'outperforms' baselines, but provides no statistical tests. On a 200-question dataset, many of the observed differences could be within random variation."
     },
     {
       "flag": "Test set contamination via hyperparameter tuning",
       "detail": "Temperature was optimized by evaluating on what appears to be the same 200 questions used for final results (Figure 1). No held-out validation or test set is described, meaning the reported accuracy may be inflated."
     },
     {
       "flag": "Wikipedia contamination risk",
       "detail": "All 200 QA pairs are derived from Bangla Wikipedia, which is almost certainly in Llama 2's training data. The model may already know the answers independently of retrieval, confounding the evaluation of the RAG system."
     },
     {
       "flag": "Overclaiming from limited evidence",
       "detail": "The abstract claims TraSe 'marks a significant advancement in RAG for low-resource languages' based on one model, one language, 200 questions, and 34% peak automatic retrieval accuracy. The 63% figure relies on human-provided oracle contexts."
     },
     {
       "flag": "No error analysis for majority failure cases",
       "detail": "The system fails on 37-66% of questions depending on configuration, but no analysis is provided of why it fails. The paper only discusses successes."
     }
   ],
   "cited_papers": [
     {
       "title": "Retrieval-Augmented Generation for Large Language Models: A Survey",
       "authors": ["Yunfan Gao", "Yun Xiong", "Xinyu Gao"],
       "year": 2023,
       "relevance": "Comprehensive survey of RAG techniques for LLMs, directly relevant to understanding the RAG landscape this paper builds upon."
     },
     {
       "title": "React: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "relevance": "Introduces the ReAct prompting framework used as a baseline in this study, relevant to agentic LLM workflows."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom B. Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational GPT-3 paper establishing few-shot prompting, which is a baseline method in this study."
     },
     {
       "title": "BanglaBERT: Language Model Pretraining and Benchmarks for Low-Resource Language Understanding Evaluation in Bangla",
       "authors": ["Abhik Bhattacharjee", "Tahmid Hasan", "Wasi Ahmad"],
       "year": 2022,
       "relevance": "Provides the BanglaBERT embedding model used in this study's retrieval pipeline, key resource for Bangla NLP."
     },
     {
       "title": "Corrective Retrieval Augmented Generation",
       "authors": ["Shi-Qi Yan", "Jia-Chen Gu", "Yun Zhu"],
       "year": 2024,
       "relevance": "Introduces retrieval quality evaluation and dynamic retrieval actions in RAG, relevant to improving RAG reliability."
     },
     {
       "title": "Ragas: Automated Evaluation of Retrieval Augmented Generation",
       "authors": ["ES Shahul", "Jithin James", "Luis Espinosa Anke"],
       "year": 2023,
       "relevance": "Proposes reference-free evaluation framework for RAG pipelines, relevant to RAG evaluation methodology."
     },
     {
       "title": "A Survey on Retrieval-Augmented Text Generation for Large Language Models",
       "authors": ["Yizheng Huang", "Xiangji Huang"],
       "year": 2024,
       "relevance": "Survey covering RAG challenges in evaluation, retrieval quality, and real-world implementation."
     },
     {
       "title": "Active Retrieval Augmented Generation (FLARE)",
       "authors": ["Zhengbao Jiang", "Frank F. Xu", "Luyu Gao"],
       "year": 2023,
       "relevance": "Introduces active retrieval mechanism for continuous information gathering during generation, advancing RAG methodology."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Potentially useful for Bangla NLP practitioners but limited by 34% accuracy and single-model evaluation."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected finding that low-resource languages need specialized approaches; no surprising results."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with existing work."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is available on GitHub, though reproducibility is limited by missing environment specs and prompts."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors from Bangladeshi universities, no well-known lab or product association."
     }
   }
 }