ai-research-survey

scan.json (27218B)

---

 {
   "paper": {
     "title": "RATHAN@DravidianLangTech 2025: Annaparavai - Separate the Authentic Human Reviews from AI-generated one",
     "authors": [
       "Jubeerathan Thevakumar",
       "Luheerathan Thevakumar"
     ],
     "year": 2025,
     "venue": "Proceedings of the Fifth Workshop on Speech, Vision, and Language Technologies for Dravidian Languages (DravidianLangTech@NAACL 2025)",
     "doi": "10.18653/v1/2025.dravidianlangtech-1.66"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "A transfer learning approach using embeddings from XLM-RoBERTa, IndicBERT, mT5, and Sentence-BERT, combined via weighted-average DNN ensemble, achieves 90% F1 for Malayalam and 73% F1 for Tamil on a shared-task AI-generated review detection dataset. Cross-validation results (94–97% F1) did not transfer to the Tamil test set, suggesting distribution mismatch between training and test data. Individual DNN models on the training split achieved up to 98.2% F1, but ensemble generalization varied by language.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in footnote 1: https://github.com/Jubeerathan/Annaparavai. The abstract and conclusion both state 'source code is publicly available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses shared task datasets from Premjith et al. (2025), which are publicly distributed through the DravidianLangTech 2025 shared task."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or library version listing is mentioned in the paper. No environment setup details beyond naming the models used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README content or reproduction steps are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report standard deviation alongside mean F1-scores for 5-fold cross-validation (e.g., Sentence-BERT: 0.962 ± 0.014 for Tamil). However, no uncertainty is reported for the DNN/ensemble results or the final test set evaluation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Comparisons between models and between ensemble vs. individual models are made purely by comparing point F1-scores."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Results are given as absolute F1-scores without contextualized comparisons (e.g., no Cohen's d, no relative improvement over a baseline)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The datasets are small (808 Tamil training, 800 Malayalam training, 100/200 test) with no justification for these sizes. No power analysis or discussion of whether sample sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation is reported across 5-fold cross-validation runs in Tables 1 and 2. However, no variance is reported for the DNN training runs or the final test set evaluation (Table 3)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No external baselines are included. The paper only compares its own model variants (four individual DNNs and an ensemble). No comparison against other shared task participants, naive baselines (e.g., majority class, random), or prior work on the same dataset."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No external baselines are included at all, so the question of whether they are contemporary is moot."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows individual model results and the ensemble result (Table 3), but does not ablate the ensemble by systematically removing components (e.g., ensemble minus one model). The individual-vs-ensemble comparison does not constitute a controlled ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only F1-score is used as the evaluation metric. Confusion matrices are shown (Figures 5–6) but precision, recall, and accuracy are not explicitly reported as separate metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this binary classification task on a labeled dataset. Automated metrics are appropriate."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The shared task provides a separate held-out test set (100 Tamil, 200 Malayalam samples). The paper also splits training data into 70/21/9 for development. Final results (73% Tamil, 90% Malayalam) are reported on the given test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by language (Tamil and Malayalam) and by embedding model (4 individual DNNs plus ensemble) in Tables 1–3. Confusion matrices show per-class performance."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes the Tamil performance drop (Section 5) and speculates about distribution mismatch, but does not show or analyze specific failure examples or error categories."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 honestly reports the significant performance drop on Tamil test data (73% vs. ~96% on CV), and acknowledges that CV performance did not transfer. This is a negative finding the authors could have downplayed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '90% f1-score for Malayalam and 73% for Tamil' which are directly supported by the test set results. The claim about 'effectiveness of transfer learning and ensembling' is supported by the results, though without external baselines the strength of that evidence is limited."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to 'demonstrat[e] the effectiveness of transfer learning and ensembling for review detection' (abstract, conclusion). Without external baselines or non-transfer-learning comparisons, the causal claim that transfer learning is what makes this effective is not justified — a simpler approach might perform similarly."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'effectiveness of transfer learning and ensembling for review detection' broadly. The conclusion says 'demonstrating the effectiveness of transfer learning in low-resource Dravidian languages' — generalizing from two languages and one task to all 'low-resource Dravidian languages.' Results are from a single shared task dataset with specific characteristics."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 briefly speculates that Tamil performance drops may be due to distribution mismatch, but does not systematically consider alternative explanations for overall results (e.g., dataset artifacts, trivial length/style differences between human and AI reviews, overfitting)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures F1-score for binary classification (human vs. AI reviews) and claims performance on this detection task. The measurements match the granularity of the claims with no proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper names 'XLM-RoBERTa', 'Indic-BERT', 'mT5', and 'Sentence-BERT' without specifying exact model sizes, checkpoint versions, or Hugging Face model IDs. For example, XLM-RoBERTa comes in base and large variants, but the paper does not specify which was used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. The pre-trained models are used solely as embedding extractors, not via prompting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for the DNN models (learning rate, batch size, epochs, optimizer, hidden layer sizes) or XGBoost (learning rate, max depth, n_estimators). The ensemble weights are also not disclosed. Figure 3 shows a DNN architecture diagram but without specific values."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a standard embedding-to-classifier pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly documents that no preprocessing was performed and explains why: the dataset consists of short, clean texts, and the language models handle tokenization internally. The data split ratios (70/21/9) are stated in Section 4.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is titled 'Limitations' and contains a substantive paragraph discussing biases inherited from language models and dataset size constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations in Section 7 are generic: 'biases inherited from the language models' and 'the dataset is limited.' These are not specific to this study — they apply to nearly any transfer-learning classification paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings are excluded. No specific boundaries like 'our results apply only to product reviews generated by [specific models]' or 'we do not claim generalization to other domains.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The datasets are from the DravidianLangTech 2025 shared task (Premjith et al., 2025) and are distributed through the shared task, making them available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 3 describes dataset sizes and labels (808/100 Tamil, 800/200 Malayalam, annotated as Human/AI) but does not describe how the data was collected — what tools generated the AI reviews, what products were reviewed, or how human reviews were sourced. This is deferred to Premjith et al. (2025)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data is from a standard shared task benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from embeddings to classification is described at a high level (Section 4.2), but exact counts at each stage are missing. The 70/21/9 split is stated as percentages but not in absolute numbers, and no information about how the split was performed (stratified? random seed?) is given."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned anywhere in the paper. No acknowledgments section is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Jubeerathan Thevakumar at University of Moratuwa, Colombo, Sri Lanka; Luheerathan Thevakumar from Jaffna, Sri Lanka. Neither evaluates a product from their employer."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed; this appears to be unfunded student/independent work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The pre-trained models (XLM-RoBERTa, IndicBERT, mT5, Sentence-BERT) are used for embedding extraction on benchmark data, but their training data cutoff dates are not stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the shared task test data could overlap with the pre-trained models' training data, or whether the training and test sets in the shared task have distributional overlap (which Section 5 hints at but does not formally analyze)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark data (AI-generated reviews) could have been seen by the pre-trained models during their training. The shared task data is presumably new, but this is not explicitly addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark classification task."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or API costs are reported. The paper does not mention wall-clock time for embedding generation or DNN training/inference."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total training time are stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity. DNN training results in Table 3 appear to be from a single run with no seed variation analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of DNN training runs is not stated. Cross-validation is 5-fold (stated), but it is unclear how many times the DNN models were trained for the results in Table 3."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The DNN architecture is shown (Figure 3) but no search over architectures or hyperparameters is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the final model configuration was selected. The paper does not describe whether the 9% validation split was used for model selection or early stopping."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper only compares its own model variants against each other. No acknowledgment that the lack of external baselines or independent evaluation could bias the assessment of effectiveness."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "No external baselines with differing compute budgets are compared, so compute-performance tradeoff analysis is not applicable."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the shared task benchmark validly measures AI-generated review detection capability. No analysis of what linguistic features the benchmark actually tests or whether performance on this benchmark transfers to real-world review detection."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The system is a standard embedding-to-classifier pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained models' training data could include information from after the shared task data creation period."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the features (embeddings) contain information that would not be available in a real deployment scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test examples share structural similarities (e.g., same AI generation models, same product domains, same review templates)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used."
       }
     }
   },
   "claims": [
     {
       "claim": "The ensemble model achieves 90% F1-score for Malayalam and 73% for Tamil on the shared task test set.",
       "evidence": "Section 5 reports these results with confusion matrices in Figures 5 and 6. The given test sets are 200 samples (Malayalam) and 100 samples (Tamil).",
       "supported": "moderate"
     },
     {
       "claim": "The weighted ensemble model outperforms individual DNN models on the training split (98.2% Tamil, 94.0% Malayalam vs. best individual model 97.1% Tamil, 96.4% Malayalam).",
       "evidence": "Table 3 shows F1-scores for all four individual DNN models and the ensemble on the 21% training split.",
       "supported": "moderate"
     },
     {
       "claim": "Transfer learning using pre-trained multilingual embeddings is effective for AI-generated review detection in low-resource Dravidian languages.",
       "evidence": "Cross-validation results in Tables 1–2 show high F1-scores (90–97%) using XGBoost on embeddings from all four models. However, no non-transfer-learning baselines are compared.",
       "supported": "weak"
     },
     {
       "claim": "The Tamil performance drop on the test set is attributable to distribution differences between training and test sets, potentially generated by different LLM models.",
       "evidence": "Section 5 notes this hypothesis but provides no empirical analysis (e.g., distribution comparison, embedding visualization) to support it.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "No external baselines",
       "detail": "The paper only compares its own model variants (four individual DNNs and an ensemble). No naive baselines (majority class, random), no prior work comparisons, and no other shared task participants' results are included. Without external baselines, claims of 'effectiveness' are unsubstantiated."
     },
     {
       "flag": "Very small test sets",
       "detail": "Tamil test set has only 100 samples and Malayalam has 200 samples. With such small test sets, F1-scores have high variance — a few misclassifications can swing results by several percentage points. No uncertainty quantification is provided for the final test results."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "No DNN hyperparameters (learning rate, batch size, epochs, optimizer, layer sizes), XGBoost hyperparameters, or ensemble weights are disclosed, making reproduction impossible even with the code."
     },
     {
       "flag": "Accuracy/F1 conflation",
       "detail": "The abstract reports '90% f1-score' but the conclusion reports '90% accuracy' for the same Malayalam result. These are different metrics, and the paper switches between them without explanation."
     },
     {
       "flag": "Large CV-to-test performance gap unexplained",
       "detail": "Tamil cross-validation F1 was 94–97% but test set F1 dropped to 73% — a ~20+ point gap. The paper attributes this to 'differences in distributions' without empirical analysis. This suggests possible overfitting or fundamental distributional mismatch that undermines the main results."
     }
   ],
   "cited_papers": [
     {
       "title": "Automatic detection of generated text is easiest when humans are fooled",
       "authors": ["Daphne Ippolito", "Daniel Duckworth", "Chris Callison-Burch", "Douglas Eck"],
       "year": 2019,
       "arxiv_id": "1911.00650",
       "relevance": "Foundational work on detecting AI-generated text using BERT-based classifiers with various decoding strategies."
     },
     {
       "title": "TweepFake: About detecting deepfake tweets",
       "authors": ["Tiziano Fagni", "Fabrizio Falchi", "Margherita Gambini", "Antonio Martella", "Maurizio Tesconi"],
       "year": 2021,
       "relevance": "Introduces sequence-based classifiers (LSTM, GRU, CNN) for detecting AI-generated social media texts."
     },
     {
       "title": "Release strategies and the social impacts of language models",
       "authors": ["Irene Solaiman", "Miles Brundage", "Jack Clark"],
       "year": 2019,
       "arxiv_id": "1908.09203",
       "relevance": "Uses RoBERTa-based classifiers to detect GPT-2-generated text, achieving ~95% accuracy."
     },
     {
       "title": "Stylometric detection of AI-generated text in Twitter timelines",
       "authors": ["Tharindu Kumarage", "Joshua Garland", "Amrita Bhattacharjee"],
       "year": 2023,
       "arxiv_id": "2303.03697",
       "relevance": "Combines stylometric features with pre-trained language models for AI-generated text detection."
     },
     {
       "title": "SeqXGPT: Sentence-level AI-generated text detection",
       "authors": ["Pengyu Wang", "Linyang Li", "Ke Ren", "Botian Jiang", "Dong Zhang", "Xipeng Qiu"],
       "year": 2023,
       "arxiv_id": "2310.08903",
       "relevance": "Uses sentence-level log probability metrics from white-box LLMs for AI-generated text detection."
     },
     {
       "title": "GPT-who: An information density-based machine-generated text detector",
       "authors": ["Saranya Venkatraman", "Adaku Uchendu", "Dongwon Lee"],
       "year": 2023,
       "arxiv_id": "2310.06202",
       "relevance": "Proposes UID-based features for distinguishing AI-generated text from human text."
     },
     {
       "title": "Neural deepfake detection with factual structure of text",
       "authors": ["Wanjun Zhong", "Duyu Tang", "Zenan Xu"],
       "year": 2020,
       "arxiv_id": "2010.07475",
       "relevance": "Combines factual structure features with RoBERTa-based classifier for AI-generated content detection."
     },
     {
       "title": "Classification of human- and AI-generated texts for different languages and domains",
       "authors": ["Kristina Schaaff", "Tim Schlippe", "Lorenz Mindner"],
       "year": 2024,
       "relevance": "Multi-language AI-generated text classification using Sentence-BERT, directly relevant to the survey's scope of AI detection methods."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The approach could be used for AI-generated review detection in Tamil/Malayalam, but the niche language scope and small shared-task context limit immediate broad applicability."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Standard transfer learning approach with expected results — no surprising findings or challenges to conventional wisdom."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Addresses AI-generated fake reviews which is a known concern for online trust, but does not reveal novel attacks or systemic risks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenging claims about other work or products."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is on GitHub, but requires specific shared-task datasets and no environment setup is provided."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Workshop paper from University of Moratuwa researchers, no major lab or product involved."
     }
   }
 }