ai-research-survey

scan-v5.json (27442B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Local LLM Ensembles for Zero-shot Portuguese Named Entity Recognition",
     "authors": [
       "João Lucas Luz Lima Sarcinelli",
       "Diego Furtado Silva"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.10043",
     "doi": "10.48550/arXiv.2512.10043"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (ensembles outperform individual models on 4/5 datasets, cross-dataset ensembles work) are directly supported by Table 3 and Figure 4 results.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation study (Table 4) removes voting and disambiguation steps to test their causal impact on performance, meeting the standard for causal claims about pipeline components.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims explicitly bounded to 'five distinct Portuguese NER datasets' and 'zero-shot' settings; future work mentions extending to other languages, indicating current scope is Portuguese-specific.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explains HAREM underperformance (generic domain, simpler entities) vs. domain-specific dataset success (complex entity types); Section 5.3 discusses validation set representativeness issues.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Measured outcome (micro-F1 on NER test sets) directly matches claimed outcome (NER performance); no conflation of proxy metrics with intended claims.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated Section 7 titled 'Limitations' discusses GPU memory requirements, processing time for evaluating 7,000+ combinations, and computational feasibility constraints.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 identifies specific threat: heuristic-selected ensemble (0.591 F1) differs from optimal ensemble (0.622 F1 on HAREM), indicating validation set may not be representative of full test distribution.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Scope explicitly bounded to 'five distinct Portuguese NER datasets' and 'zero-shot' prompting; future work implies extension to other languages is outside current scope.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states: 'financed in part by the Coordenação de Aperfeiçoamento de Pessoal de Nível Superior – Brasil (CAPES) – Finance Code 001.'",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors listed with affiliation 'Instituto de Ciências Matemáticas e Computação, Universidade de São Paulo' and no conflict with evaluated model organizations (Meta, Google, Microsoft, etc.).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "CAPES is a Brazilian government research funding agency, independent of the paper's outcomes regarding ensemble performance.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement provided; no mention of patents, equity, consulting fees, or financial relationships with model publishers.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms established: NER as token-level task (standard in literature), ensemble defined via voting/disambiguation steps, zero-shot as 'no labeled examples,' in-context learning cited with reference [24].",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions listed in Introduction: (1) novel three-step ensemble pipeline, (2) evaluation on 5 Portuguese datasets, (3) evidence that cross-dataset ensembles work.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 positions work vs. MoE models, fusion-based ensembles (LLM-Blender), routing approaches (FrugalGPT), prior LLM NER work [13,16], showing novel contribution in zero-shot Portuguese NER with similarly-sized models.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Abstract and Section 3 state 'Code is available at https://github.com/Joao-Luz/local-llm-ner-ensemble' with direct GitHub link provided.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All five evaluation datasets (HAREM, LeNER-Br, UlyssesNER-Br, GeoCorpus-2, MariNER) are publicly available standard benchmarks with citations to original papers.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Table 2 provides exact model versions with HuggingFace links and temperatures specified (t=0, 1.0, 1.5), but no requirements.txt, dependencies, or Python version documented in the paper.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Pipeline steps described in Section 3 with figures, but actual prompts for extraction and voting steps not provided in full (only structure shown in Figures 1-2); no step-by-step reproducibility guide in paper.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Table 3 caption states 'averaged of three runs' to account for stochasticity, but no error bars, standard deviations, or confidence intervals reported in results.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests (t-tests, ANOVA) reported for performance comparisons between ensemble and individual models; claims based only on point estimates.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": false,
           "justification": "Absolute micro-F1 scores provided but effect sizes not explicitly reported; e.g., ensemble improvement on LeNER-BR (0.541 vs 0.490) shown as raw numbers without effect size quantification.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "Validation set uses 100 random sentences per dataset with justification only as 'simulating a low-resource scenario'; no power analysis or sample size calculation provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Only averages over three runs shown in Table 3; no standard deviations, variances, or confidence intervals across the runs reported.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 includes baseline results for all individual models (LLaMA3, Qwen2, Gemma2, Phi3, Mistral) and supervised RoBERTa model.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Model baselines are contemporary (LLaMA3/Qwen2/Gemma2/Phi3 from 2024, Mistral from 2023); RoBERTa (2019) is standard supervised baseline.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 ablates the voting step (no voting) and simplifies disambiguation, showing impact of each component across all datasets.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": false,
           "justification": "Only micro-F1 reported as evaluation metric; no macro-F1, precision, recall, or per-entity-type metrics provided despite datasets having multiple entity types.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "No human evaluation needed for NER task evaluated on gold-standard annotated datasets; not applicable.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.1 states 'available train split to perform ensemble configuration selection and test split for testing'; results in Table 3 are on held-out test sets.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": false,
           "justification": "Datasets (Table 1) specify multiple entity types per domain, but results (Table 3) only report overall micro-F1 without per-entity-type or per-category performance breakdowns.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "HAREM underperformance (ensemble 0.591 vs individual 0.609) analyzed in Section 5.1 with explanation; GeoCorpus2 heuristic brittleness discussed in Section 5.3.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "HAREM results (ensemble underperforms individual models) reported honestly in Table 3 and analyzed rather than omitted.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 specifies exact model names (LLaMA 3.1-8B, Qwen2-7B, Gemma-2-9B, Phi-3-Medium-128K, Mistral-7B) with HuggingFace links and parameter counts.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "Section 3 describes prompt structure ('task definition entity type descriptions...') and Figure 2 shows disambiguation prompt construction, but actual zero-shot prompts for extraction and voting steps not provided in full.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Temperature values for each model in extraction step explicitly reported (t=0 for LLaMA3/Mistral/Phi3, t=1.0 for Gemma2, t=1.5 for Qwen2); voting/disambiguation at t=0.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Three-step pipeline (extraction, voting, disambiguation) described in detail in Section 3 with mechanistic explanation of each step and Figures 1-2 illustrating the process.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 describes validation set selection (100 random sentences per dataset) and entity parsing ('validated... to remove those with inconsistent types or those that cannot be matched').",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All five datasets (HAREM, LeNER-Br, UlyssesNER-Br, GeoCorpus-2, MariNER) are publicly available with citations to original papers in Table 1.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "While not collecting new data, Table 1 provides dataset descriptions with citations to original papers that document collection procedures.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable—using existing annotated datasets, not recruiting participants.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 documents dataset selection, train/test split usage, validation set creation (100-sentence sample), and processing pipeline steps (extraction → voting → disambiguation).",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Model training cutoffs not stated; models listed with 2024 or 2023 release dates but no explicit knowledge cutoff documentation provided.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether Portuguese NER datasets (especially 2025's MariNER) might appear in model training data; Portuguese is lower-resource but potential overlap not addressed.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Not addressed; no discussion of whether evaluation datasets were in training data or possibility of benchmark leakage.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "applies": false,
         "answer": false
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Section 7 mentions 'high GPU memory requirements' and 'long processing times' qualitatively, but no concrete inference latency, tokens per second, or cost metrics reported.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Section 7 states 'over 7,000 combinations' must be evaluated for configuration selection, but no total computational budget (GPU-hours, etc.) provided.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Ensembles outperform individual LLMs for Portuguese NER on 4 out of 5 datasets",
       "evidence": "Table 3 shows ensemble F1 scores vs individual model F1 across LeNER-BR (0.541 vs 0.490), GeoCorpus2 (0.313 vs 0.270), UlyssesNER (0.353 vs 0.340), and MariNER (0.706 vs 0.699). HAREM is the exception (0.591 ensemble vs 0.609 best individual Gemma2).",
       "supported": "moderate"
     },
     {
       "claim": "Different models excel at different pipeline stages (extraction, voting, disambiguation)",
       "evidence": "Section 5.1 analysis shows LLaMA3/Gemma2 dominate as extractors (appear in 4/5 dataset extraction configs), while Phi3 appears mainly in voting/disambiguation (section 5.1: 'Phi3 shows up for most voting and disambiguation ensembles').",
       "supported": "moderate"
     },
     {
       "claim": "The voting step is critical to ensemble performance",
       "evidence": "Table 4 ablation shows removing voting step causes significant F1 drops: HAREM 0.592→0.410, LeNER-BR 0.541→0.478, GeoCorpus2 0.313→0.178, UlyssesNER 0.353→0.249, MariNER 0.706→0.621.",
       "supported": "strong"
     },
     {
       "claim": "Cross-dataset ensemble configurations perform as well as target-dataset ensembles",
       "evidence": "Figure 4 shows HAREM and UlyssesNER ensembles trained on other datasets outperform heuristic-best ensemble from same target dataset (text: 'ensemble configurations obtained in cross-dataset configurations may perform better than ones obtained from heuristic applied to same dataset').",
       "supported": "moderate"
     },
     {
       "claim": "The heuristic for selecting ensemble configuration is robust across train/test splits",
       "evidence": "Section 5.3 shows heuristic selects 0.591 F1 on HAREM test but optimal possible ensemble achieves 0.622 F1 (Figure 3), indicating validation set selection does not reliably predict test performance.",
       "supported": "weak"
     },
     {
       "claim": "Zero-shot LLM ensembles enable NER for lower-resource languages without fine-tuning",
       "evidence": "Paper evaluates all models zero-shot (no fine-tuning) on 5 Portuguese datasets. Results show ensembles competitive with individual models without labeled fine-tuning data.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "The paper proposes a three-step ensemble pipeline (extraction → voting → disambiguation) for zero-shot Portuguese NER using five similarly-sized open-weight LLMs. Ensembles outperform individual models on 4 of 5 datasets (F1 gains most pronounced on domain-specific datasets with complex entity types), though a heuristic for configuration selection shows limited robustness: validation-set selection (0.591 F1 HAREM) underperforms the globally optimal ensemble (0.622 F1). Crucially, ensemble configurations trained on different source datasets often match or exceed target-dataset performance, enabling practical zero-shot NER without task-specific annotation.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "Results presented as point estimates (averages of 3 runs) without confidence intervals, standard deviations, or significance tests despite inherent variance in LLM outputs."
     },
     {
       "flag": "Heuristic selection brittleness",
       "detail": "Section 5.3 demonstrates that validation-set-based heuristic (0.591 F1) substantially underperforms globally optimal ensemble (0.622 F1 on HAREM), questioning robustness of configuration selection method."
     },
     {
       "flag": "Single evaluation metric",
       "detail": "Only micro-F1 reported; no macro-F1, precision, recall, or per-entity-type breakdowns despite datasets having 4-16 entity types each, limiting understanding of performance gaps."
     },
     {
       "flag": "Unjustified sample size",
       "detail": "100-sentence validation set used without power analysis or justification; section 5.3 hints this may be unrepresentative of full test distribution."
     },
     {
       "flag": "Training data contamination not addressed",
       "detail": "No discussion of whether Portuguese NER datasets (especially 2025's MariNER, released same year as paper) might be in model training data; Portuguese corpora less likely to be excluded than English but not verified."
     },
     {
       "flag": "Prompts under-specified",
       "detail": "Actual zero-shot prompts for extraction and voting steps not fully provided; only structural descriptions in Section 3 and Figure 2, limiting reproducibility."
     },
     {
       "flag": "No per-entity-type analysis",
       "detail": "Despite datasets having 4-16 entity types (Table 1), results report only overall micro-F1, preventing analysis of which entity types benefit from ensembling."
     },
     {
       "flag": "Supervision leakage in zero-shot claim",
       "detail": "Configuration selection uses 100 labeled sentences per dataset (section 4.3), technically violating 'zero-shot' framing; though minimized, ensemble is supervised in selection phase."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-NER: Named Entity Recognition via Large Language Models",
       "authors": "Wang et al.",
       "year": 2023,
       "relevance": "Directly addresses zero-shot NER with LLMs; prior work establishing LLM underperformance on token-level tasks."
     },
     {
       "title": "Empirical Study of Zero-Shot NER with ChatGPT",
       "authors": "Xie et al.",
       "year": 2023,
       "relevance": "Baseline zero-shot NER evaluation methodology; compares zero-shot prompting strategies."
     },
     {
       "title": "LLM Chain Ensembles for Scalable and Accurate Data Annotation",
       "authors": "Farr et al.",
       "year": 2024,
       "relevance": "Ensemble routing approach for task-specific model selection; alternative aggregation strategy."
     },
     {
       "title": "Improving entity recognition using ensembles of deep learning and fine-tuned large language models: A case study on adverse event extraction",
       "authors": "Li et al.",
       "year": 2025,
       "relevance": "Recent LLM ensemble for NER; uses fine-tuning and larger models (GPT-3.5), contrasts with this paper's zero-shot approach."
     },
     {
       "title": "An Ensemble of LLMs Finetuned with LoRA for NER in Portuguese Legal Documents",
       "authors": "Nunes et al.",
       "year": 2025,
       "relevance": "Portuguese NER ensemble; most closely related work; uses fine-tuning vs. zero-shot, legal domain only."
     },
     {
       "title": "A Review of Hybrid and Ensemble in Deep Learning for Natural Language Processing",
       "authors": "Jia, Liang, Liang",
       "year": 2024,
       "relevance": "Comprehensive ensemble methods survey; provides taxonomies of fusion-based and routing strategies."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": "Chen, Zaharia, Zou",
       "year": 2023,
       "relevance": "Routing approach for model selection; cost-aware ensemble strategy alternative to voting."
     },
     {
       "title": "A Survey of Large Language Models",
       "authors": "Zhao et al.",
       "year": 2025,
       "relevance": "Foundational LLM capabilities and limitations survey; contextualizes zero-shot performance gap."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Useful for Portuguese NER practitioners but requires GPU resources to run 5 models in parallel; no production inference optimization or model distillation discussed."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Ensemble improvement is expected; the main surprise (HAREM underperformance on simple domain) is noted but not deeply investigated or challenged."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety concerns raised, addressed, or relevant to NER task."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild tension in Section 5.3 (heuristic fails to find optimal ensemble), but insufficient for social engagement."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub; datasets are public; but reproducibility requires GPU setup and full prompt specifications are missing from paper."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from USP (reputable but not top-tier AI lab); uses well-known open models (Meta LLaMA, Google Gemma) but no partnership or industry affiliation."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45745607",
         "title": "Collatz-Weyl Generators: Pseudorandom Number Generators (2023)",
         "points": 59,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=45745607",
         "created_at": "2025-10-29T11:55:42Z"
       },
       {
         "hn_id": "38794757",
         "title": "Knowledge Graph Reasoning Based on Attention GCN",
         "points": 52,
         "comments": 10,
         "url": "https://news.ycombinator.com/item?id=38794757",
         "created_at": "2023-12-28T16:02:24Z"
       },
       {
         "hn_id": "46287626",
         "title": "Detailed balance in large language model-driven agents",
         "points": 48,
         "comments": 5,
         "url": "https://news.ycombinator.com/item?id=46287626",
         "created_at": "2025-12-16T12:17:08Z"
       },
       {
         "hn_id": "38722880",
         "title": "ReST Meets ReAct: Self-Improvement for Multi-Step Reasoning LLM Agent",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=38722880",
         "created_at": "2023-12-21T16:45:46Z"
       },
       {
         "hn_id": "38681247",
         "title": "Point Transformer V3: Simpler, Faster, Stronger",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=38681247",
         "created_at": "2023-12-18T11:29:41Z"
       },
       {
         "hn_id": "46279856",
         "title": "Detailed balance in large language model-driven agents",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46279856",
         "created_at": "2025-12-15T20:11:28Z"
       },
       {
         "hn_id": "45930419",
         "title": "A Large-Scale Computational Analysis of Errors in ArXiv Papers",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45930419",
         "created_at": "2025-11-14T18:52:29Z"
       },
       {
         "hn_id": "45927607",
         "title": "Black-Box On-Policy Distillation of Large Language Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45927607",
         "created_at": "2025-11-14T15:18:22Z"
       },
       {
         "hn_id": "43204606",
         "title": "Strassen Multisystolic Array Hardware Architectures",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43204606",
         "created_at": "2025-02-28T11:48:06Z"
       },
       {
         "hn_id": "42418107",
         "title": "Towards Reasoning in Large Language Models: A Survey (2023)",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42418107",
         "created_at": "2024-12-14T16:50:49Z"
       }
     ],
     "top_points": 59,
     "total_points": 169,
     "total_comments": 18
   }
 }