ai-research-survey

scan.json (31083B)

---

 {
   "paper": {
     "title": "Are LLMs Prescient? A Continuous Evaluation using Daily News as the Oracle",
     "authors": ["Hui Dai", "Ryan Teehan", "Mengye Ren"],
     "year": 2024,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2411.08324",
     "doi": "10.48550/arXiv.2411.08324"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM forecasting accuracy degrades significantly over time, with average declines of 21.55% on True/False and 11.33% on Multiple Choice questions between Jan 2020 and Dec 2024. Degradation accelerates after models' knowledge cutoff dates. RAG partially mitigates this but the declining trend persists, and even gold article access (reading comprehension) shows performance decline, suggesting outdated internal representations compound the missing-knowledge problem.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Abstract states 'Code and data are available at https://agenticlearning.ai/daily-oracle.' A URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Same URL provides dataset access. The paper describes 31,510 QA pairs and states code and data are available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or specific library versions in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions found in the paper. Prompts are provided in the appendix but no instructions for running the full evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 and all figures report point estimates only (accuracy percentages and YoY changes). No confidence intervals, error bars, or uncertainty measures on any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'performance degradation' and model comparisons are made by comparing raw accuracy numbers. No statistical significance tests (p-values, t-tests, etc.) are used anywhere in the paper."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage changes with baseline context: 'performance declines by 21.55% on TF questions (from 64.68% to 50.74%) and by 11.33% on MC questions (from 58.30% to 51.69%)' (Section 4.2). Table 3 provides per-model YoY changes with absolute yearly accuracies."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset has 31,510 QA pairs (16,783 TF + 14,727 MC) and the human evaluation uses 60 pairs, but no justification for why these sizes are sufficient and no power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be single-run evaluations (each model answers each question once). No variance, standard deviation, or spread measures across runs are reported. 5-month moving averages are used for smoothing but this is not variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Eight LLMs are compared against each other across three evaluation settings (closed-book, constrained open-book, gold article). The closed-book setting serves as baseline for the RAG settings."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models include Claude-3.5-Sonnet, GPT-4, Llama-3-8B, Qwen-2-7B, Gemma-2-2B — all recent models at time of writing. Also includes older models (GPT-3.5, Mistral-7B) for temporal contrast."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation settings (closed-book, constrained open-book with varying RAG cutoffs, gold article) systematically ablate the effect of external knowledge access. Multiple RAG cutoff dates isolate the effect of information recency."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results reported separately for TF accuracy and MC accuracy, plus YoY accuracy change (pre-cutoff and post-cutoff), refusal rates, and temporal regression slopes (Figure 6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation (Section 3.3) validates the quality of generated QA pairs, not the LLM forecasting outputs. No human evaluation of the models' forecasting performance is performed — evaluation is entirely automated accuracy scoring."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Models are pre-trained and not fine-tuned on any part of the Daily Oracle dataset. The entire benchmark functions as a held-out test set. Temporal structure provides natural separation between training data and evaluation data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by question type (TF vs MC), per model, per year (Table 3), pre-cutoff vs post-cutoff, and by evaluation setting. Category distribution shown in Figure 1b and Figure 8. Appendix provides per-model constrained open-book results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Refusal behavior analyzed in detail (Section 4.3, Appendix B.2, Figures 9-11). Figure 20 shows a concrete failure case example with Mixtral-8x7B giving wrong answer in closed-book but correct in open-book. Performance drops below random baseline discussed for Mistral-7B and Mixtral-8x7B."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RAG sometimes hurts performance: 'Llama-3-8B may perform worse than the closed-book setting when the RAG cutoff is prior to the knowledge cutoff dates' (Section 4.2). Claude-3.5-Sonnet open-book 'lags behind its closed-book performance' (Appendix B.4). Gold article setting still shows declining trends despite providing the answer."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of performance degradation (21.55% TF, 11.33% MC) are directly supported by Table 3 and Section 4.2. Claims that RAG helps but degradation persists are supported by Figure 4 and Section 4.2. All abstract claims have corresponding results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims outdated pre-training data causes performance degradation. The study design uses temporal structure as a natural quasi-experiment, with three evaluation settings (closed-book, open-book, gold article) to disentangle missing knowledge from outdated representations. The pre/post-cutoff comparison and gold article finding provide reasonable causal support."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title asks 'Are LLMs Prescient?' and abstract claims about 'LLMs' temporal generalization and forecasting abilities' broadly, but the benchmark covers only English news from 5 specific sources (CBS, CNBC, CNN, Forbes, NPR) with LLM-generated questions. Results may not generalize to other languages, domains, or question formats. These scope boundaries are not explicitly stated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses multiple alternative explanations: missing knowledge vs outdated representations, overrepresentation of pre-September 2021 data in training corpora, websites restricting web crawlers after ChatGPT. The Limitations section adds question generation bias and event selection bias."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures accuracy on LLM-generated news QA pairs and frames this as 'forecasting ability' and 'temporal generalization.' It does not discuss whether answering auto-generated True/False and Multiple Choice questions about news events is a valid proxy for actual forecasting ability, which involves reasoning under uncertainty, probability calibration, and handling of novel scenarios."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 (Appendix B.1) lists exact model versions: claude-3-5-sonnet-20240620, gpt-4-1106-preview, gpt-3.5-turbo-0125, Mixtral-8x7B-Instruct-v0.1, Mistral-7B-Instruct-v0.3, Meta-Llama-3-8B-Instruct, Qwen2-7B-Instruct, gemma-2-2b-it."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts for all steps are provided in Appendix D (Figures 23-37): article summary, QA generation, misleading choices generation, QA filtering, and all evaluation prompts for closed-book, open-book, and gold article settings for both TF and MC."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference hyperparameters (temperature, top-p, max tokens) are reported for any of the evaluated models. RAG uses BM25 with top-5 retrieval and 512-word truncation, but LLM sampling parameters are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated via direct prompting without tool use, retry logic, or multi-step reasoning frameworks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The 4-step construction pipeline is documented in detail (Section 3.1, Figure 7): article selection (3 random + 3 hot topic via DBSCAN clustering), article summarization, QA generation with few-shot prompting, misleading choice generation, and QA filtering with 7 principles and 13-point threshold. Article corpus details given (1,246,973 articles from Jan 2019 to Dec 2024)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' subsection at the end of Section 4.3 discusses data generation biases and evaluation limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: generated questions may contain biases from an outdated LLM generator, questions only cover events that definitively occurred (not non-events that never made the news), limited time horizon after each model's cutoff date prevents thorough post-cutoff analysis, and the RAG/knowledge cutoff interaction is weak within available data."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Limitations section mentions specific issues, it does not explicitly state what the results do NOT show. The paper does not explicitly bound claims to English news from 5 sources, does not state that results may not generalize to other forecasting domains, languages, or question formats."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code and data are available at the provided URL, and the dataset of 31,510 QA pairs is described as released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection described in detail: Common Crawl News Dataset + Newspaper3k scraping, filtered to 5 mainstream sources (CBS, CNBC, CNN, Forbes, NPR), 1,246,973 English articles from Jan 2019 to Dec 2024. Article selection strategy and QA generation pipeline documented."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Four human annotators are used for quality evaluation (Section 3.3) but no information is provided about who they are, how they were recruited, or their qualifications. This could introduce bias in the quality assessment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: news corpus collection → article selection (random + hot topic) → 4-step QA construction (summarization → generation → misleading choices → filtering with 7 criteria and 13-point threshold). The dataset size at each stage is not fully specified (e.g., how many articles were filtered out at each step), but the pipeline stages and criteria are clearly described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section lists: IITP grants from Republic of Korea (No. RS-2024-00469482 & RS-2024-00509279), Microsoft Accelerating Foundation Models Research program for Azure credits, and NYU HPC."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed as affiliated with New York University. They evaluate third-party models (GPT, Claude, Llama, etc.) — no direct affiliation with evaluated model companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft provided Azure cloud compute credits for LLM APIs, and the paper evaluates GPT models (Microsoft/OpenAI products). Microsoft has a financial interest in GPT model performance. This potential conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 lists knowledge cutoff dates for models: Claude-3.5-Sonnet (Apr 2024), GPT-4 (Apr 2023), GPT-3.5 (Sept 2021), Llama-3-8B (Mar 2023), Gemma-2-2B (Jul 2024). Models with unknown cutoffs are marked as 'Unknown.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is central to the paper. The entire experimental design is built around studying how performance changes before and after training cutoffs. Section 4.3 discusses pre-September 2021 data overrepresentation in training corpora and its effects."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is designed to mitigate contamination by construction: questions are generated from daily news after most models' training cutoffs. Section 4.3 discusses that earlier questions may be contaminated by training data and analyzes the performance cliff around September 2021. The pre/post-cutoff analysis explicitly addresses this."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The 4 human annotators validate dataset quality but are not experimental subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper evaluates LLMs on a benchmark, not human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example costs reported. The paper mentions switching to GPT-4o-mini/GPT-4o for cost-effectiveness and 'budget constraints' preventing some experiments, but actual costs are not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated. Acknowledgments mention NYU HPC and Microsoft Azure credits, but no GPU hours, total API spend, or hardware specifications are provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or repeated runs. Results appear to be single evaluations per model per question. No seed sensitivity analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It appears each model answered each question once, but this is not explicitly confirmed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "RAG parameters (top-5 retrieval, 512-word truncation, BM25) and QA filtering threshold (13 points) are stated but no justification for these choices or search budget is reported. LLM inference hyperparameters are not mentioned at all."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Fixed configurations are used without justification: BM25 with top-5, 512-word truncation, 13-point QA filter threshold. No explanation of how these specific values were selected."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing third-party models on a benchmark rather than proposing a new model to compare against baselines. No self-comparison bias in the traditional sense."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes are compared (Gemma-2-2B vs GPT-4, Claude-3.5-Sonnet) without discussing compute budget differences. No performance-vs-compute analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper engages with construct validity through multiple evaluation settings: closed-book tests forecasting, gold article tests reading comprehension, and open-book tests RAG-augmented forecasting. Section 4.3 disentangles missing knowledge from outdated representations. Appendix C compares with forecasting market datasets to argue their approach better captures temporal trends."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. Models are evaluated via direct prompting without any agentic scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Central to the paper's design. Knowledge cutoff dates are used to separate pre-training data from evaluation data. RAG cutoff dates prevent information leakage from future articles. The entire paper studies how temporal information boundaries affect performance."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "QA filtering principle 3 (absence of information leakage) explicitly checks that questions don't reveal post-publication information. The RAG cutoff formally limits accessible articles to before min(dres−1, dR-Cutoff). The construction ensures questions don't contain answer-leaking temporal information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether QA pairs from the same news article or topic cluster are correlated. Figure 2 shows topic frequency is temporally autocorrelated, suggesting questions across days are not independent. This non-independence could affect the reliability of temporal trend estimates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Temporal splits based on knowledge cutoff dates are used as the primary leakage prevention method. The QA filtering pipeline includes an automated LLM-based check for information leakage (principle 3), and the RAG cutoff mechanism formally prevents future information access."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM performance on forecasting questions degrades over time, with average declines of 21.55% on TF and 11.33% on MC questions between January 2020 and December 2024.",
       "evidence": "Table 3 and Figure 3 show yearly accuracy for 8 models. Average accuracy drops from 64.68% to 50.74% (TF) and 58.30% to 51.69% (MC) across all models (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Performance degradation accelerates after models' knowledge cutoff dates, with post-cutoff YoY declines steeper than pre-cutoff declines.",
       "evidence": "Table 3 shows GPT-4 MC post-cutoff YoY decline of -18.54% vs pre-cutoff -4.23%. Claude-3.5-Sonnet post-cutoff drops -12.41% (TF) and -11.78% (MC) vs pre-cutoff -4.77% and -6.26%. Figure 6 shows regression slope becoming more negative after cutoffs.",
       "supported": "strong"
     },
     {
       "claim": "RAG improves prediction accuracy but the performance degradation pattern persists across all RAG cutoff settings.",
       "evidence": "Figure 4 shows Mixtral-8x7B TF accuracy improves with later RAG cutoffs, but declining trends persist in all curves. Llama-3-8B MC shows similar patterns. Section 4.2 notes RAG can even hurt performance when cutoff is before knowledge cutoff.",
       "supported": "strong"
     },
     {
       "claim": "Even with gold article access (reading comprehension), LLM performance declines over time, suggesting outdated internal representations beyond just missing knowledge.",
       "evidence": "Figure 5 shows most models approach ~90% accuracy with gold articles but still exhibit declining trends. Section 4.3 discusses this as evidence that 'the models' internal representations are outdated.'",
       "supported": "moderate"
     },
     {
       "claim": "Daily Oracle is the largest and most up-to-date continuously updated forecasting benchmark.",
       "evidence": "Table 1 compares with 10 prior benchmarks. Daily Oracle has 31,510 questions with daily updates, larger and more frequently updated than ForecastBench (1,000, biweekly), FreshBench (2,769), and ForecastQA (10,382, static).",
       "supported": "strong"
     },
     {
       "claim": "The LLM-based QA filtering achieves 85% accuracy compared to human consensus on final QA pair acceptance.",
       "evidence": "Section 3.3 and Table 4: 4 human annotators evaluate 60 QA pairs. Average accuracy across 7 principles is 89.52%, and final acceptance accuracy is 85.00%. Fleiss' Kappa of 0.26 indicates fair agreement among annotators.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All performance comparisons and degradation claims are based on comparing raw accuracy numbers without any statistical tests. With thousands of questions per time period, significance tests would be straightforward and would strengthen the claims."
     },
     {
       "flag": "Single-run evaluation with no variance estimation",
       "detail": "Each model appears to answer each question once. No multiple runs, no seed sensitivity, no bootstrap confidence intervals. The smoothing via 5-month moving averages is not a substitute for proper uncertainty quantification."
     },
     {
       "flag": "Weak human evaluation agreement",
       "detail": "The human annotator agreement is only 'fair' (Fleiss' Kappa = 0.26). Some principles have near-zero agreement (Non-answerability: 0.02). This raises questions about the reliability of the QA filtering criteria and the benchmark quality validation."
     },
     {
       "flag": "LLM-generated benchmark evaluated on LLMs",
       "detail": "GPT-3.5/GPT-4 generate the QA pairs, and GPT-3.5/GPT-4 are among the evaluated models. This creates potential circular biases — questions may be easier or harder for the generating model family. This is not discussed."
     },
     {
       "flag": "Missing inference hyperparameters",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any evaluated model. These significantly affect output and could explain some performance variation."
     },
     {
       "flag": "Undisclosed Microsoft conflict",
       "detail": "Microsoft provided Azure compute credits and the paper evaluates GPT models (Microsoft/OpenAI). This potential conflict is not acknowledged, though results don't appear to favor GPT models."
     }
   ],
   "cited_papers": [
     {
       "title": "NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark",
       "authors": ["O. Sainz", "J. Campos", "I. García-Ferrero", "J. Etxaniz", "O. L. de Lacalle", "E. Agirre"],
       "year": 2023,
       "relevance": "Directly relevant to benchmark contamination — argues every LLM benchmark must assess data contamination, which is a core concern this paper addresses."
     },
     {
       "title": "Benchmark data contamination of large language models: A survey",
       "authors": ["C. Xu", "S. Guan", "D. Greene", "M.-T. Kechadi"],
       "year": 2024,
       "arxiv_id": "2406.04244",
       "relevance": "Comprehensive survey of benchmark contamination in LLMs, directly relevant to the contamination and temporal evaluation themes."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus", "F. Petroni"],
       "year": 2020,
       "relevance": "Foundational RAG paper — the constrained open-book evaluation setting in this paper directly builds on RAG methodology."
     },
     {
       "title": "Approaching human-level forecasting with language models",
       "authors": ["D. Halawi", "F. Zhang", "C. Yueh-Han", "J. Steinhardt"],
       "year": 2024,
       "relevance": "Evaluates LLM forecasting capabilities using forecasting market questions — directly comparable to Daily Oracle's approach."
     },
     {
       "title": "Inadequacies of large language model benchmarks in the era of generative artificial intelligence",
       "authors": ["T. R. McIntosh", "T. Susnjak", "N. Arachchilage"],
       "year": 2025,
       "relevance": "Critiques LLM benchmarks for lacking temporal dimensions and becoming outdated — the same problems Daily Oracle aims to address."
     },
     {
       "title": "Task contamination: Language models may not be few-shot anymore",
       "authors": ["C. Li", "J. Flanigan"],
       "year": 2024,
       "relevance": "Demonstrates task contamination in few-shot LLM evaluation, supporting the need for dynamic benchmarks."
     },
     {
       "title": "ForecastBench: A dynamic benchmark of AI forecasting capabilities",
       "authors": ["E. Karger", "H. Bastani", "C. Yueh-Han", "Z. Jacobs", "D. Halawi", "F. Zhang", "P. E. Tetlock"],
       "year": 2025,
       "relevance": "Dynamic forecasting benchmark that Daily Oracle directly compares against — biweekly updates with 1,000 questions from forecasting markets."
     },
     {
       "title": "Mind the gap: Assessing temporal generalization in neural language models",
       "authors": ["A. Lazaridou", "A. Kuncoro", "E. Gribovskaya"],
       "year": 2021,
       "relevance": "Defines temporal generalization for language models and demonstrates performance deterioration over time — foundational work for this paper's research question."
     },
     {
       "title": "Is your LLM outdated? A deep look at temporal generalization",
       "authors": ["C. Zhu", "N. Chen", "Y. Gao", "Y. Zhang", "P. Tiwari", "B. Wang"],
       "year": 2025,
       "relevance": "Introduces FreshBench and studies LLM temporal generalization — closest comparable work using forecasting market questions."
     },
     {
       "title": "Consent in crisis: The rapid decline of the AI data commons",
       "authors": ["S. Longpre", "R. Mahari", "A. Lee"],
       "year": 2024,
       "relevance": "Documents increasing restrictions on web crawlers post-ChatGPT, which the paper cites as a potential explanation for training data imbalance across time periods."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["A. Dubey", "A. Jauhri", "A. Pandey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Llama-3-8B is one of the eight evaluated models in the benchmark experiments."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "GPT-4 is one of the evaluated models and used for QA generation in the benchmark construction pipeline."
     }
   ]
 }