ai-research-survey

scan.json (23647B)

---

 {
   "paper": {
     "title": "Towards Temporal Knowledge-Base Creation for Fine-Grained Opinion Analysis with Language Models",
     "authors": ["Gaurav Negi", "Atul Kr. Ojha", "Omnia Zayed", "Paul Buitelaar"],
     "year": 2025,
     "venue": "KBC-LM @ ISWC 2025",
     "arxiv_id": "2509.02363",
     "doi": "10.48550/arXiv.2509.02363"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes a DSPy-based LLM annotation pipeline for creating temporal opinion knowledge bases using three fine-grained opinion data models (ACOS, SSA, UOC). Best F1 scores on human-annotated test sets are 59.92 (ACOS), 45.91 (SSA), and 58.17 (UOC) using 8B parameter models. Inter-LLM annotation agreement varies widely across opinion components, with classification labels (sentiment polarity) showing higher agreement than span-based elements (entity, aspect terms).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub URL is provided: https://github.com/ANON-1221/KBC-LM-Temporal-Opinions for the annotated dataset. However, this appears to be an anonymized URL, which may not resolve. The paper states 'The annotated dataset is available publicly' at that URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (ACOS, SSA/SemEval 2022 Task 10, UOC, StockMotions, Politifact) and claims to release the annotated knowledge base at the GitHub URL."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using SGLang for serving and an NVIDIA RTX 4090, but provides no requirements.txt, Dockerfile, or library versions beyond the model names."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The pipeline description is high-level (DSPy programs, MIPRO optimizer) without runnable commands or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3 and 4 are point estimates (P, R, F1) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares models and settings (e.g., Llama vs Ministral, COT vs no COT) but no statistical significance tests are applied. Differences are discussed informally."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw F1 scores are reported but no effect sizes (e.g., Cohen's d) or contextualized magnitude measures are provided for comparisons between settings."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The training sample sizes (150, 152, 30 examples for DSPy) and test sample sizes are stated but not justified. No power analysis or discussion of whether these sizes are adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures across runs are reported. Temperature is set to 0.0 for determinism, but DSPy optimization involves stochastic elements (MIPRO) and no repeated-run variance is shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Zero-shot performance serves as a baseline, compared against MIPRO-optimized settings with varying ICL examples (5, 10, 15 shot). Two models are also compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare against any prior LLM-based annotation methods or state-of-the-art opinion extraction systems. The only baselines are zero-shot versions of the same models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies ICL examples (0, 5, 10, 15) and COT (yes/no) across both models, which functions as an ablation of pipeline components (Table 3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Precision, recall, and F1 are all reported for each configuration in Table 3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the LLM-generated annotations is performed beyond comparison to existing human-annotated test sets. The temporal KB annotations are only evaluated via inter-LLM agreement, not human judgment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates training samples (for DSPy/ICL configuration) from test samples for evaluation. Table 1 shows separate train/eval/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 breaks down by data model (SSA, ACOS, UOC). Table 4 provides per-component agreement (Target, Holder, Aspect Term, Polarity, etc.) across data models and datasets."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses low-agreement components (entity, aspect terms) and Section 6.2 provides qualitative examples showing where different data models produce different or inconsistent annotations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that COT reasoning 'does not seem to have a definitive effect' and shows cases where more ICL examples hurt performance (e.g., SSA with 15-shot COT at 39.52 vs 10-shot COT at 45.91)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'scalable method' for temporal KB creation with 'rigorous quantitative evaluation' and 'inter-annotator agreement.' Tables 3 and 4 support these claims. The claims are appropriately scoped."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are modest — the ablation design (varying ICL count, COT) with controlled single-variable manipulation is adequate for the claims made about pipeline configuration effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly limits scope to two 8B models and acknowledges in Limitations (Section 8) that results may improve with larger or proprietary models. Claims are bounded to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for observed results. For example, low inter-LLM agreement on spans could be due to model architecture differences, training data differences, or inherent task ambiguity — none of these are explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses F1 between two LLMs as a 'proxy for IRR' without discussing the gap between inter-LLM agreement and actual annotation quality. Two models could agree on wrong answers. This proxy limitation is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model identifiers are provided: 'mistralai/Ministral-8B-Instruct-2410' and 'meta-llama/Llama-3.1-8B-Instruct' (Section 5, footnotes 3-4)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses DSPy for prompt optimization but does not provide the actual optimized prompts or the DSPy signatures used. Only the pipeline structure is described at a high level."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 reports temperature=0.0, context window=128K, output length=4096, and training sample sizes for DSPy compilation (150, 152, 30)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The pipeline is a straightforward DSPy program with LLM API calls, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the outlier estimation and stratified sampling protocol in detail, including IQR-based upper bounds and the rationale for excluding outliers. Figure 5 visualizes the distributions."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is titled 'Limitations and Future Work' and discusses three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 mentions specific limitations: only 8B parameter models used, only open-weight models tested, and lack of annotation ensembling. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states what was not tested: larger models, proprietary models, and annotation ensembling methods. The scope is bounded to the two tested models."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The annotated knowledge base is claimed to be publicly available at the GitHub URL. The source datasets (StockMotions, Politifact, ACOS, SSA, UOC) are also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides temporal coverage and sizes of the KB datasets. Table 1 describes the data model datasets with domain, split sizes. The source datasets are referenced with citations."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from existing public datasets (StockMotions from StockTwits, Politifact from fact-checking)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 documents the three-stage pipeline (configuration → evaluation → annotation). Section 4.2 details the sampling protocol with outlier exclusion criteria and stratified sampling."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 (Acknowledgments) discloses SFI Grant Number SFI/12/RC/2289_P2 (Insight_2) and Research Ireland Postdoctoral Fellowship GOIPD/2023/1556."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Insight SFI Research Ireland Centre for Data Analytics, University of Galway, clearly stated in the header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is Science Foundation Ireland, a public research funding agency with no commercial interest in the paper's outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates Llama-3.1-8B and Ministral-8B on opinion extraction tasks using existing annotated datasets but does not state the training data cutoff dates for either model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The ACOS and SSA datasets are publicly available and could be in the training data of both models. This overlap risk is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The test datasets (ACOS from ACL 2021, SSA from SemEval 2022) predate both models' training and were likely available online. Contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported despite running annotation pipelines over ~31K examples across multiple settings."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using an NVIDIA RTX 4090 but provides no GPU hours, wall-clock time, or total compute budget for pipeline configuration, evaluation, or annotation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Temperature is 0.0 for LLM inference, but DSPy MIPRO optimization has stochastic components and no seed sensitivity is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It appears results are from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The MIPRO optimizer is used to select prompts and ICL examples, but the search budget (number of configurations explored, compute spent) is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically reports all tested configurations in Table 3 (zero-shot, 5/10/15-shot, with/without COT) and selects the best for temporal annotation based on test set F1."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across models, settings, and data models with no statistical tests at all, let alone corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own pipeline without acknowledging potential bias in their implementation or evaluation choices."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different ICL settings have different compute costs (more examples = more tokens) but performance is not analyzed as a function of compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses component-level exact match F1 as its metric and discusses why tuple-level exact match is too harsh (citing Schaeffer et al.), but does not discuss whether F1 on these benchmarks actually measures annotation quality for downstream temporal analysis tasks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; the DSPy pipeline is the method being tested, not a scaffold wrapping a model."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The ACOS (2021) and SSA (2022) benchmark datasets predate both models' training. The paper does not discuss whether models may have memorized these datasets."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ICL examples are drawn from the same dataset distribution as test examples. No discussion of whether this introduces information leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Train and test splits come from the same datasets. No discussion of potential structural similarities or non-independence between training examples for DSPy and the test set."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "The DSPy-based declarative annotation pipeline achieves best F1 of 59.92 on ACOS, 45.91 on SSA, and 58.17 on UOC against human-annotated test sets.",
       "evidence": "Table 3 reports these scores: ACOS best is Llama-3.1-8B with 15-shot no COT (59.92), SSA best is Llama-3.1-8B 10-shot with COT (45.91), UOC best is Ministral-8B 5-shot with COT (58.17).",
       "supported": "moderate"
     },
     {
       "claim": "The framework eliminates the need for manual prompt engineering when annotating fine-grained opinions with LLMs.",
       "evidence": "Section 4 describes the DSPy pipeline where MIPRO automatically optimizes prompts and selects ICL examples. However, DSPy signatures and program structures still require manual design.",
       "supported": "weak"
     },
     {
       "claim": "Classification labels (sentiment polarity, intensity) show higher inter-LLM agreement than span-based components (entity, aspect terms).",
       "evidence": "Table 4 shows sentiment polarity agreement at 67-78% across models/datasets, while entity agreement is 14-21% and aspect terms 14-37%.",
       "supported": "strong"
     },
     {
       "claim": "COT reasoning does not have a definitive effect on performance across the tested models.",
       "evidence": "Table 3 shows mixed results: COT helps in zero-shot settings on average but sometimes hurts in few-shot settings (e.g., ACOS Llama 15-shot: 59.92 no-COT vs 56.69 with COT).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "All comparisons between models, ICL settings, and COT configurations are made by comparing point estimates with no significance tests, confidence intervals, or variance measures. Small F1 differences (e.g., 45.91 vs 45.69) are discussed as meaningful."
     },
     {
       "flag": "Inter-LLM agreement used as quality proxy",
       "detail": "For the temporal KB annotations, quality is assessed only by agreement between two 8B LLMs. Two models could systematically agree on incorrect annotations. No human validation of the temporal KB is performed."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "Both models were likely trained on the publicly available ACOS and SSA datasets. Without decontamination analysis, the pipeline evaluation results may be inflated."
     },
     {
       "flag": "Anonymized GitHub URL",
       "detail": "The dataset release URL (github.com/ANON-1221/KBC-LM-Temporal-Opinions) appears to be an anonymized placeholder that may not resolve to an actual repository."
     }
   ],
   "cited_papers": [
     {
       "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
       "authors": ["Omar Khattab", "Arnav Singhvi", "Paridhi Maheshwari"],
       "year": 2024,
       "relevance": "Core framework used for the LLM annotation pipeline; relevant to LLM programming methodology."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational work on LLM few-shot capabilities that motivates the annotation approach."
     },
     {
       "title": "Large Language Models for Data Annotation and Synthesis: A Survey",
       "authors": ["Zhen Tan", "Dawei Li", "Song Wang"],
       "year": 2024,
       "doi": "10.18653/v1/2024.emnlp-main.54",
       "relevance": "Survey of LLM-based data annotation methods directly relevant to AI-assisted annotation quality assessment."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Discusses discontinuous evaluation metrics and emergent abilities — relevant to LLM capability evaluation methodology."
     },
     {
       "title": "Optimizing Instructions and Demonstrations for Multi-Stage Language Model Programs",
       "authors": ["Krista Opsahl-Ong", "Michael J. Ryan", "Josh Purtell"],
       "year": 2024,
       "doi": "10.18653/v1/2024.emnlp-main.525",
       "relevance": "MIPRO optimizer used in the annotation pipeline; relevant to LLM prompt optimization methodology."
     },
     {
       "title": "ProSA: Assessing and Understanding the Prompt Sensitivity of LLMs",
       "authors": ["Jingming Zhuo", "Songyang Zhang", "Xinyu Fang"],
       "year": 2024,
       "doi": "10.18653/v1/2024.findings-emnlp.108",
       "relevance": "Addresses LLM prompt sensitivity, which motivates the declarative DSPy approach used in this work."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "COT prompting technique evaluated as a pipeline component in this work."
     }
   ]
 }