ai-research-survey

scan.json (27271B)

---

 {
   "paper": {
     "title": "Local LLM Ensembles for Zero-shot Portuguese Named Entity Recognition",
     "authors": [
       "João Lucas Luz Lima Sarcinelli",
       "Diego Furtado Silva"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.10043",
     "doi": "10.48550/arXiv.2512.10043"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "A three-step ensemble pipeline (extraction, voting, disambiguation) using 5 locally-run LLMs (7-14B parameters) outperforms individual models on 4 of 5 Portuguese NER datasets in zero-shot settings. The voting step is critical to performance, while disambiguation can sometimes be simplified. Cross-dataset ensemble configurations generally transfer well, often matching or exceeding configurations selected on the target dataset, suggesting the pipeline is viable even without any annotated data for the target task.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided in the abstract and Section 3: https://github.com/Joao-Luz/local-llm-ner-ensemble."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All five datasets used (HAREM, LeNER-Br, UlyssesNER-Br, GeoCorpus-2, MariNER) are publicly available with citations to their original publications in Table 1."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of Python version, CUDA version, GPU model, library versions, requirements.txt, or Dockerfile. The paper mentions 'consumer GPUs' in Section 7 but does not specify which hardware or software environment was used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A code repository is linked but the paper does not describe how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 reports only point estimates of micro-F1. Although Section 5.1 states results are averaged over three runs, no confidence intervals, error bars, or ± notation are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ensembles outperform individual LLMs based on comparing F1 scores in Table 3 without any statistical significance tests (no p-values, t-tests, or other tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports absolute micro-F1 scores for both individual models and ensembles, providing sufficient context to assess the magnitude of improvement (e.g., ensemble 0.541 vs best individual 0.528 on LeNER-BR test)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The validation set size of 100 sentences per dataset (Section 4.3) is not justified via power analysis or other reasoning beyond 'simulating a low-resource scenario.'"
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 states 'the reported results are the average of three runs of each experiment' but no standard deviation, IQR, or other spread measure is reported anywhere. The reader cannot assess result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 includes individual LLM baselines (each model running the full pipeline alone) and a fully supervised RoBERTa model trained on full train splits."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines use contemporary models: LLaMA 3.1 (2024), Qwen2 (2024), Gemma 2 (2024), Phi3 (2024), Mistral v0.2 (2023), and RoBERTa. All are recent and appropriate."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 and Table 4 present an ablation study removing the voting step and simplifying the disambiguation step, demonstrating the contribution of each pipeline component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only micro-F1 is reported as the evaluation metric. No precision, recall, or other complementary metrics are provided."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated using micro-F1 against gold-standard annotations. No human evaluation of the NER outputs is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly separates the train split (100-sentence sample for configuration selection) from the test split used for final evaluation. 'Testing is performed in the entire test set available for each dataset.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-dataset breakdowns across all 5 datasets. Individual model performance is also broken down per dataset."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses HAREM as the dataset where the ensemble underperformed individual models. Section 5.3 analyzes limitations of the heuristic selection method with Figure 3."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "HAREM underperformance is reported in Section 5.1. Section 5.3 shows the heuristic doesn't find the optimal configuration. The overall honest reporting of when ensembles fail to help is commendable."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of outperforming individual LLMs in 4/5 datasets are supported by Table 3. The cross-dataset transferability claim is supported by Figure 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation (Section 5.2) that voting and disambiguation contribute to performance. The ablation design uses controlled removal of single components, which is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states 'our pipeline is not dependent on this language' without testing on any other language. The abstract frames contributions broadly ('scalable, low-resource, and zero-shot NER') based on only Portuguese data with 5 specific models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers hypotheses for why ensembles help on domain-specific datasets (Section 5.1) but does not systematically consider confounds or alternative explanations such as data characteristics, annotation quality differences across datasets, or model-specific biases."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures micro-F1 on NER tasks and claims NER performance. The measurement matches the claim granularity — no proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 footnotes link to specific HuggingFace models: Meta-Llama-3.1-8B-Instruct, Qwen2-7B-Instruct, gemma-2-9b-it, Phi3-Medium-128K-Instruct, Mistral-7B-Instruct-v0.2. These are specific versioned model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts in natural language only — e.g., 'zero-shot prompt strategy adapted from [22]', 'task definition entity type descriptions.' Actual prompt text is not provided in the paper. A code repository is linked which may contain the prompts, but the paper itself shows no prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 reports temperature values for each model: t=0 for LLaMA3, Mistral, Phi3; t=1.0 for Gemma2; t=1.5 for Qwen2. Voting and disambiguation use t=0. The calibration procedure is described."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a three-step prompting pipeline (extraction, voting, disambiguation) without tools, retry logic, memory, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes entity validation (removing inconsistent types, matching to original sentence). Section 4.3 describes the 100-sentence sampling from train splits and the temperature calibration procedure using HAREM."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations' is a dedicated section discussing GPU memory requirements, processing time, and the computational burden of evaluating 7,000+ combinations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 discusses only practical/computational limitations (GPU memory, processing time). No specific threats to validity are discussed — e.g., the representativeness of the 100-sentence validation sample, the generalizability beyond Portuguese, or the stability of results across runs."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The conclusion claims language independence ('our pipeline is not dependent on this language') without bounding this. No explicit statement of what populations, languages, model sizes, or settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All five NER datasets are publicly available: HAREM [18], LeNER-Br [15], UlyssesNER-Br [2], GeoCorpus-2 [4], MariNER [19]. Each is cited with its original publication."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Table 1 describe the datasets used, including their domains, entity types, and provenance. Section 4.3 describes the sampling and splitting procedure."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public NER benchmark datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from data input to final evaluation is documented: dataset selection → 100-sentence sampling for validation → temperature calibration → model combination exhaustive search → test set evaluation. Entity validation steps described in Section 3.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discloses funding from CAPES (Coordenação de Aperfeiçoamento de Pessoal de Nível Superior – Brasil) with Finance Code 001."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are from Instituto de Ciências Matemáticas e Computação, Universidade de São Paulo. They do not evaluate their own products — all LLMs are from third parties (Meta, Google, Microsoft, Alibaba, Mistral AI)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "CAPES is a Brazilian government agency for academic research funding with no financial stake in the specific LLM or NER outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five LLMs used (LLaMA 3.1, Qwen2, Gemma 2, Phi3, Mistral v0.2). The models are evaluated on NER benchmarks they may have seen during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the NER datasets (some published as early as 2006 for HAREM) appeared in any of the models' training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HAREM (2006), LeNER-Br (2018), GeoCorpus-2 (2020), and UlyssesNER-Br (2022) all predate the training of the models used. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 mentions 'high GPU memory requirements' and 'long processing times' but does not quantify inference cost, latency, tokens consumed, or wall-clock time for any experiment."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total compute budget are stated. Section 7 notes '7,000 combinations' must be processed but does not report how long this took or what hardware was used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 3 runs (Section 5.1) but no standard deviation, variance, or sensitivity analysis across seeds is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly states: 'the reported results are the average of three runs of each experiment.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The search is exhaustive over 7,000+ model combinations (Section 7), but no compute budget (GPU hours, wall-clock time) for the search is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 describes selection of the best configuration by micro-F1 on a 100-sentence validation sample from the train split. The criterion and procedure are clearly stated."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates many model combinations across 5 datasets but applies no statistical tests, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose and evaluate their own ensemble method against individual model baselines without acknowledging the bias of evaluating their own system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The ensemble uses multiple LLMs (3-5 models per step) with significantly more compute than individual model baselines. This cost-performance tradeoff is not analyzed. The supervised RoBERTa baseline uses different compute (full training) with no comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether micro-F1 on these NER benchmarks actually measures the intended NER capability, or whether the benchmarks have construct validity issues."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. All models are evaluated within the same three-step prompting pipeline framework."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Several benchmarks (HAREM 2006, LeNER-Br 2018, GeoCorpus-2 2020) significantly predate the LLMs' training periods, creating contamination risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The zero-shot prompts provide entity type definitions which could vary in how much information they convey."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test splits share structural similarities (e.g., same documents, same domains) that could inflate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "The ensemble pipeline outperforms individual LLMs in 4 out of 5 Portuguese NER datasets.",
       "evidence": "Table 3 shows ensemble configurations achieving higher micro-F1 than any individual model on LeNER-BR (0.541 vs 0.528), GeoCorpus2 (0.313 vs 0.262), UlyssesNER (0.353 vs 0.334), and MariNER (0.706 vs 0.699). HAREM is the exception where Gemma2 individually (0.609) outperforms the ensemble (0.591).",
       "supported": "moderate"
     },
     {
       "claim": "The voting step is critical to ensemble performance.",
       "evidence": "Table 4 (Section 5.2) shows consistent performance drops when voting is removed: HAREM -0.181, LeNER-BR -0.063, GeoCorpus2 -0.135, UlyssesNER -0.104, MariNER -0.085 in micro-F1.",
       "supported": "strong"
     },
     {
       "claim": "Cross-dataset ensemble configurations generally outperform individual LLMs, potentially eliminating the need for annotated data.",
       "evidence": "Figure 4 shows that ensembles obtained on different source datasets outperform individual LLMs for 4/5 target datasets. For HAREM and UlyssesNER, cross-dataset configurations even outperformed the heuristic-selected configuration.",
       "supported": "moderate"
     },
     {
       "claim": "The heuristic for selecting model combinations may not find the optimal ensemble configuration.",
       "evidence": "Section 5.3 and Figure 3 show that for HAREM, the overall best ensemble (Gemma2 extraction, Phi3 voting, LLaMA3 disambiguation) outperforms the heuristic-selected ensemble, indicating the heuristic is sensitive to validation set size and representativeness.",
       "supported": "strong"
     },
     {
       "claim": "Individual models complement each other's knowledge gaps for domain-specific entity types.",
       "evidence": "Section 5.1 hypothesizes that domain-specific datasets (GeoCorpus2 with 13 types, UlyssesNER with 16 types, LeNER-BR with 6 types) benefit more from ensembles because individual models struggle with specialized entities but complement each other. However, this is not tested directly.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "Results are averaged over 3 runs but no standard deviation or confidence intervals are reported. With only 3 runs and no spread measures, the reader cannot assess whether differences between models/ensembles are meaningful or within noise."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (ensemble vs individual models) are based on comparing point estimates without any statistical tests. Some differences are very small (e.g., MariNER ensemble 0.706 vs Gemma2 0.699) and may not be statistically significant."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Pre-trained LLMs are evaluated on NER benchmarks dating back to 2006 (HAREM). The models may have seen these datasets during pre-training, which would inflate zero-shot performance and undermine the paper's core premise."
     },
     {
       "flag": "Small validation set for configuration selection",
       "detail": "The heuristic selects from 7,000+ configurations using only 100 sentences. Section 5.3 demonstrates this is unreliable (heuristic-selected configuration is suboptimal for HAREM), but no analysis is provided for other datasets."
     },
     {
       "flag": "Single evaluation metric",
       "detail": "Only micro-F1 is reported. Precision and recall are not shown, making it impossible to understand whether ensembles improve coverage (recall) or accuracy (precision) or both."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-NER: Named Entity Recognition via Large Language Models",
       "authors": ["S. Wang", "X. Sun", "X. Li", "R. Ouyang", "F. Wu", "T. Zhang", "J. Li", "G. Wang"],
       "year": 2023,
       "arxiv_id": "2304.10428",
       "doi": "10.48550/arXiv.2304.10428",
       "relevance": "Directly evaluates LLM capabilities for NER, showing they underperform specialized models — core motivation for the ensemble approach."
     },
     {
       "title": "LLM-Blender: Ensembling Large Language Models with Pairwise Ranking and Generative Fusion",
       "authors": ["D. Jiang", "X. Ren", "B. Y. Lin"],
       "year": 2023,
       "arxiv_id": "2306.02561",
       "doi": "10.48550/arXiv.2306.02561",
       "relevance": "Key prior work on LLM ensembling through fusion-based methods, demonstrating that combining multiple LLMs can outperform individuals."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "doi": "10.48550/arXiv.2305.05176",
       "arxiv_id": "2305.05176",
       "relevance": "Proposes cost-effective LLM usage through routing and cascading, directly relevant to the survey's coverage of LLM efficiency and routing strategies."
     },
     {
       "title": "LLM Chain Ensembles for Scalable and Accurate Data Annotation",
       "authors": ["D. Farr", "N. Manzonelli", "I. Cruickshank", "K. Starbird", "J. West"],
       "year": 2024,
       "doi": "10.1109/BigData62323.2024.10826109",
       "relevance": "Proposes cascading LLM chains for classification tasks, demonstrating routing and ensemble approaches for practical LLM deployment."
     },
     {
       "title": "Mixtral of Experts",
       "authors": ["A. Q. Jiang", "A. Sablayrolles", "A. Roux"],
       "year": 2024,
       "arxiv_id": "2401.04088",
       "relevance": "Mixture-of-experts LLM architecture that uses ensemble of trainable weight subsets, relevant to the survey's coverage of LLM architectures and efficiency."
     },
     {
       "title": "Improving entity recognition using ensembles of deep learning and fine-tuned large language models: A case study on adverse event extraction from VAERS and social media",
       "authors": ["Y. Li", "D. Viswaroopan", "W. He", "J. Li", "X. Zuo", "H. Xu", "C. Tao"],
       "year": 2025,
       "doi": "10.1016/j.jbi.2025.104789",
       "relevance": "Uses LLM ensembles including GPT-3.5 for NER in the medical domain, showing ensemble voting outperforms individual models."
     },
     {
       "title": "Empirical Study of Zero-Shot NER with ChatGPT",
       "authors": ["T. Xie", "Q. Li", "J. Zhang", "Y. Zhang", "Z. Liu", "H. Wang"],
       "year": 2023,
       "arxiv_id": "2310.10035",
       "doi": "10.48550/arXiv.2310.10035",
       "relevance": "Empirical evaluation of ChatGPT's zero-shot NER capabilities, providing the prompt strategy adapted by this paper."
     },
     {
       "title": "A Survey of Large Language Models",
       "authors": ["W. X. Zhao", "K. Zhou", "J. Li"],
       "year": 2025,
       "arxiv_id": "2303.18223",
       "doi": "10.48550/arXiv.2303.18223",
       "relevance": "Comprehensive LLM survey covering in-context learning capabilities referenced for establishing LLM baseline capabilities."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["A. Grattafiori", "A. Dubey", "A. Jauhri"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "doi": "10.48550/arXiv.2407.21783",
       "relevance": "Technical report for one of the five open-weight LLMs evaluated, relevant to the survey's coverage of open-weight model capabilities."
     },
     {
       "title": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone",
       "authors": ["M. Abdin", "J. Aneja", "H. Awadalla"],
       "year": 2024,
       "arxiv_id": "2404.14219",
       "relevance": "Technical report for the Phi-3 model used in the ensemble, relevant to small/local LLM capabilities."
     }
   ]
 }