ai-research-survey

scan.json (17829B)

---

 {
   "paper": {
     "title": "Context-Alignment: Activating and Enhancing LLM Capabilities in Time Series",
     "authors": ["Yuxiao Hu", "Qian Li", "Dongxiao Zhang", "Jinyue Yan", "Yuntian Chen"],
     "year": 2025,
     "venue": "ICLR 2025",
     "arxiv_id": "2501.03747"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/tokaka22/ICLR25-FSCA, mentioned in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used (ETT, M4, UEA, Weather, Electricity, Traffic, ILI) are publicly available standard benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions PyTorch and NVIDIA H800/RTX 4090 GPUs (Appendix A.1) but does not provide a requirements.txt, Dockerfile, or detailed library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README-level instructions are described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., MSE values) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims FSCA 'outperforms' and 'surpasses' baselines based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baseline context are reported, e.g., '3.1% MSE reduction over PatchTST', '7.3%, 12.2%, and 16.6%' over LLM-based methods (Sec. 4.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why these specific datasets or dataset sizes were chosen beyond following prior work conventions."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or multi-run statistics are reported. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines included: GPT4TS, Time-LLM, S2IP-LLM, iTransformer, PatchTST, DLinear, TimesNet, FEDformer, ETSformer, N-HiTS, N-BEATS (Sec. 4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods like iTransformer (2023), S2IP-LLM (2024), Time-LLM (2024), PatchTST (2022), which are recent and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive ablation study in Sec. 4.7 and Table 6: removes dual-scale GNNs, coarse-grained branch, varies layer count and insertion positions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "MSE and MAE for long-term/few-shot/zero-shot forecasting; SMAPE, MASE, and OWA for short-term; accuracy for classification."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for time series forecasting/classification tasks evaluated by automated metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard train/test splits from established benchmarks are used. Zero-shot evaluation trains on one dataset and tests on another (Sec. 4.5)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down per dataset (8 datasets for long-term, 6 M4 subsets for short-term, 10 UEA datasets for classification) and per prediction horizon."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is discussed. The paper only presents cases where FSCA succeeds."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Ablation study reports configurations that hurt performance: random initialization (A.2), removing coarse-grained branch (B.1), excessive layers (C.5, C.6) causing overfitting."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of effectiveness across tasks, especially few-shot and zero-shot forecasting, are supported by Tables 2-6 showing consistent improvements."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'Context-Alignment activates LLM capabilities' are supported by ablation studies (Table 6) showing controlled single-variable manipulation (removing GNNs, coarse branch, etc.)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim general 'LLM Capabilities in Time Series' but results are exclusively with GPT-2 as the backbone. No other LLMs are tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the improvements are discussed. The paper does not consider whether gains could come from added parameters, GNN architecture itself independent of alignment, or other confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-2' without specifying which version/size (small, medium, large, xl). Appendix A.1 mentions 'pre-trained models from Wolf et al. [2020]' but no specific model variant."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Actual prompt text is provided: 'Predict future sequences using previous data:' (Sec. 3.2) and 'Predict category (x in total) using previous data:' (Appendix B.1)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 reports Adam optimizer, decay rates (0.9, 0.999), learning rates, cosine annealing schedule (Tmax=20, eta_min=10^-8), batch size 256, N=2, early stopping, loss functions."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard deep learning method."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 3.1 describes patching with sliding window of size p and stride s. Dataset details including dimensions, lengths, and frequencies are in Appendix A.2 Tables 7-9."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do not show. The paper does not acknowledge that results are limited to GPT-2 or specific dataset types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available standard benchmarks (ETT, M4, UEA, Weather, Electricity, Traffic, ILI)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Dataset details documented in Appendix A.2 with Tables 7-9 describing length, dimensions, frequency, and sources for all datasets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; all data comes from standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw time series to patches to embeddings is documented in Sec. 3.1 (Token Embedding) and the experimental setup follows established protocols from Wu et al. [2022]."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 6 (Acknowledgement) lists NSFC Grant No. 62106116, China Meteorological Administration grant, National Key R&D Program, and Ningbo Major Science and Technology Projects."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed: Hong Kong Polytechnic University, Eastern Institute of Technology Ningbo, Shanghai Jiao Tong University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are government research agencies (NSFC, China Meteorological Administration, National Key R&D Program) with no financial stake in the specific outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses GPT-2 as a backbone with frozen/fine-tuned layers for time series tasks, not evaluating the LLM's pretrained knowledge on a benchmark. Contamination of text training data is not relevant to time series evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Time series benchmarks are numerical data, not subject to LLM text training data contamination. Standard train/test splits are used."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation is on time series numerical data, not text benchmarks. LLM text contamination is not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported despite the method adding GNN computation on top of LLM inference."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (H800, RTX 4090) but no total GPU hours, training time, or computational budget is stated."
       }
     }
   },
   "claims": [
     {
       "claim": "FSCA reduces average MSE by 3.1% over PatchTST and outperforms LLM-based methods (S2IP-LLM, Time-LLM, GPT4TS) by 7.3%, 12.2%, and 16.6% on long-term forecasting.",
       "evidence": "Table 2 (Sec. 4.2) shows MSE/MAE across 8 datasets and multiple horizons.",
       "supported": "moderate"
     },
     {
       "claim": "FSCA achieves 13.3% improvement over PatchTST in zero-shot forecasting.",
       "evidence": "Table 5 (Sec. 4.5) shows cross-domain transfer results on ETT datasets.",
       "supported": "moderate"
     },
     {
       "claim": "FSCA achieves 6.7% MSE reduction over S2IP-LLM in few-shot forecasting with 5% training data.",
       "evidence": "Table 4 (Sec. 4.4) with results on ETT datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Context-Alignment paradigm is the first to propose context-level alignment between time series and language.",
       "evidence": "Stated in contributions (Sec. 1) and related work (Sec. 2.2), contrasting with token-level alignment methods.",
       "supported": "weak"
     },
     {
       "claim": "FSCA achieves 76.4% average accuracy on UEA classification, a 2.4% increase over the next best model.",
       "evidence": "Figure 2 (Sec. 4.6) shows average accuracy across 10 UEA datasets.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes Context-Alignment, a paradigm that aligns time series data with linguistic components via Dual-Scale Context-Alignment GNNs (structural and logical alignment) to activate LLM capabilities for time series tasks. FSCA, which integrates few-shot prompting with this framework, achieves consistent improvements over baselines across long-term, short-term, few-shot, and zero-shot forecasting on standard benchmarks, with particularly strong gains in data-scarce settings. Ablation studies confirm that both dual-scale structure and logical alignment via directed edges contribute to performance.",
   "red_flags": [
     {
       "flag": "No variance or multi-run statistics",
       "detail": "All results are single-run point estimates with no error bars, standard deviations, or confidence intervals, making it impossible to assess whether differences are statistically meaningful."
     },
     {
       "flag": "Single LLM backbone",
       "detail": "All experiments use GPT-2 only, but the paper claims general 'LLM Capabilities in Time Series' without testing on any other LLM."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity discussion despite several obvious limitations (single LLM, no statistical testing, no cost analysis)."
     },
     {
       "flag": "No significance testing",
       "detail": "Claims of 'outperforming' and 'surpassing' are based solely on comparing point estimates without any statistical tests."
     }
   ],
   "cited_papers": [
     {
       "title": "Time-LLM: Time Series Forecasting by Reprogramming Large Language Models",
       "authors": ["Ming Jin", "Shiyu Wang", "Lintao Ma"],
       "year": 2024,
       "relevance": "Key baseline for LLM-based time series forecasting using reprogramming approach."
     },
     {
       "title": "One Fits All: Power General Time Series Analysis by Pretrained LM",
       "authors": ["Tian Zhou", "Peisong Niu", "Liang Sun", "Rong Jin"],
       "year": 2023,
       "relevance": "GPT4TS baseline demonstrating frozen pretrained transformers for time series analysis."
     },
     {
       "title": "S2IP-LLM: Semantic Space Informed Prompt Learning with LLM for Time Series Forecasting",
       "authors": ["Zijie Pan", "Yushan Jiang", "Sahil Garg"],
       "year": 2024,
       "relevance": "Key baseline for semantic space alignment between LLMs and time series."
     },
     {
       "title": "TEST: Text Prototype Aligned Embedding to Activate LLM's Ability for Time Series",
       "authors": ["Chenxi Sun", "Hongyan Li", "Yaliang Li", "Shenda Hong"],
       "year": 2024,
       "relevance": "Token-level alignment approach for activating LLMs on time series tasks."
     },
     {
       "title": "TEMPO: Prompt-based Generative Pre-trained Transformer for Time Series Forecasting",
       "authors": ["Defu Cao", "Furong Jia", "Sercan O Arik"],
       "year": 2024,
       "relevance": "Prompt-based approach for LLM time series forecasting using decomposition."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom B Brown"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "Foundation work on few-shot prompting that inspired the FSCA approach."
     },
     {
       "title": "UniTS: Building a Unified Time Series Model",
       "authors": ["Shanghua Gao", "Teddy Koker", "Owen Queen"],
       "year": 2024,
       "arxiv_id": "2403.00131",
       "relevance": "Unified time series model relevant to LLM-based time series analysis landscape."
     }
   ]
 }