ai-research-survey

scan.json (24016B)

---

 {
   "paper": {
     "title": "Towards Fundamental Language Models: Does Linguistic Competence Scale with Model Size?",
     "authors": ["Jaime Collado-Montañez", "L. Alfonso Ureña-López", "Arturo Montejo-Ráez"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.02225",
     "doi": "10.48550/arXiv.2509.02225"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Linguistic competence stabilizes at moderate model sizes (5-7B parameters) while internal factual knowledge continues to scale steeply with model size. Linear regression shows model size explains 81% of variance in internal factual knowledge but only 50% for linguistic competence, with a slope twice as steep for factual knowledge. Mann-Whitney U tests show no statistically significant improvement from medium to large models for any competence. Results support the proposed Fundamental Language Model paradigm of smaller linguistically competent models with external factual retrieval.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or download link is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used (WiC, BLiMP, RTE, MNLI, QQP, LAMBADA, BoolQ, COPA, MultiRC, ReCoRD, TriviaQA, TruthfulQA) are publicly available standard benchmarks, and evaluation was done through the publicly available LM Evaluation Harness framework."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Every experiment has been executed on two NVIDIA Ampere A100 GPUs' (Section 4) but provides no software versions, library versions, or environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, scripts, or step-by-step guide are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 3-6 are reported as point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Mann-Whitney U tests are used for pairwise comparisons between size categories (Section 4.1.2, Table 2), with p-values reported at α=0.05."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage performance differences (e.g., 'IFK shows a 39.50% performance difference between large and small models, while Linguistic Competence and EFK only shows 18.29% and 8.47% respectively') and R² values (0.81 vs 0.50) and regression slopes (0.059 vs 0.029) in Section 4.1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 23 models were selected or whether this sample size is adequate for the statistical analyses performed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be single-run evaluations. No standard deviations, variance, or multiple-run results are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple model families across different sizes serve as comparisons (SmolLM2, Qwen2.5, Llama-3, OLMo-2, Falcon3, Gemma-2, Yi-1.5), though there is no 'baseline method' per se since this is a comparative scaling study."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include recent releases: OLMo-2-0325, Qwen2.5, Llama-3.2, Falcon3, Gemma-2 — all from 2024-2025."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a comparative scaling study evaluating existing models, not proposing a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple benchmarks are used within each competence category: 5 benchmarks for linguistic competence (WiC, BLiMP, RTE, MNLI, QQP), 5 for external factual knowledge, and 4 for internal factual knowledge."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here; the paper measures model performance on established benchmarks with automated metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets are used via the LM Evaluation Harness framework, which uses the established test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-task breakdowns are provided in Tables 3-6 (linguistic subcompetences, semantic tasks, external FK tasks, internal FK tasks) in addition to aggregate scores."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative discussion of where or why specific models fail on specific benchmarks. Only aggregate scores are analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that medium-to-large improvements are not statistically significant (Table 2), and that external factual knowledge does not improve with scale beyond a threshold, which are effectively negative findings about scaling."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'internal factual knowledge grows significantly faster' are supported by the regression analysis (R²=0.81 vs 0.50, slope 0.059 vs 0.029) and that 'linguistic competence and factual knowledge improve with scale' is shown in Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-adjacent claims like 'model size is more closely tied to memorization than to core language ability' (abstract) and that model size 'drives' performance differences. However, model families differ in architecture and training data, so model size is confounded with these variables. The paper acknowledges this in limitations but the main text does not hedge the causal framing."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only English benchmarks but proposes a general 'Fundamental Language Model' paradigm without bounding claims to English. The abstract says 'a path toward more efficient, interpretable, and sustainable NLP solutions' broadly. The limitations acknowledge this but the main claims are unbounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section discusses that architecture and training data decisions matter (not just size), that language-factual boundaries are hard to draw, and that results may not generalize across languages or tasks. Section 4 also notes architecture/training matter: 'depending more on architecture and training data decisions than on model size.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines 'linguistic competence' using the CEFR framework (Section 1), maps it to specific sub-competences (lexical, grammatical, semantic), and maps each to specific benchmarks (Section 3.1). The proxy-outcome relationship is transparent — benchmark scores proxy for linguistic competence as defined by CEFR."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are given (e.g., 'SmolLM2-135M', 'Qwen2.5-0.5B', 'OLMo-2-0325-32B', 'Llama-3.2-1B'). These include version identifiers (e.g., OLMo-2-0325 vs OLMo-2-1124). However some like 'gemma-2-2b' lack snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'zero-shot setting' and uses the LM Evaluation Harness but does not provide the actual prompt templates used for each benchmark."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, decoding strategy) are reported for the model evaluations."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; models are evaluated directly on benchmarks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how benchmark data was preprocessed or how the LM Evaluation Harness formatted inputs. The paper references 'standard evaluation protocols' without specifying them."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the conclusions, discussing English-only evaluation, architecture dependence, and linguistic-factual boundary challenges."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section raises specific threats: English-only benchmarks may not generalize, the linguistic-factual boundary is hard to draw ('understanding metaphors, cultural references, or domain-specific terminology often depends on both'), and findings may not extend to other architectures."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly states what was not tested: other languages, other architectures, real-world deployment, and hybrid approaches. It also acknowledges that phonological/orthoepic/orthographic competences are excluded (Section 1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw evaluation outputs, per-example predictions, or underlying data are made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes in detail which benchmarks were selected for each competence and why, with citations for each benchmark. The evaluation framework (LM Evaluation Harness) is identified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; all data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw benchmark outputs to the aggregated scores in Tables 1, 3-6 is not documented. For example, how ReCoRD's F1 and exact match are averaged, or how TruthfulQA Generation's multiple metrics are combined, is mentioned briefly but the full pipeline is not made reproducible."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists multiple Spanish government grants (CONSENSO, MODERATES, SocialTOX), an FPI scholarship, and EU NextGenerationEU funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with the Computer Science Department, University of Jaén, Spain. No evaluated product is their own."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from Spanish government and EU research grants, which have no financial stake in whether smaller models are linguistically competent."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 23 models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any benchmark data (e.g., WiC, BLiMP, MNLI) appeared in the training data of the models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Many benchmarks used (BLiMP 2020, MNLI 2018, WiC 2019, TriviaQA 2017) predate all models and are widely available online. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or time per evaluation is reported despite running 23 models across 14 benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only 'two NVIDIA Ampere A100 GPUs' is mentioned. No total GPU hours, wall-clock time, or compute budget is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per model-benchmark pair is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "This is an evaluation study using pre-trained models with the LM Evaluation Harness defaults; no hyperparameter tuning is performed."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection is performed; models are evaluated with default settings via the LM Evaluation Harness."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Nine Mann-Whitney U tests are performed (3 competences × 3 pairwise comparisons, Table 2) without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The authors do not propose a system; they evaluate existing models on public benchmarks. Self-comparison bias does not apply."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Performance is plotted against model size (parameter count) but not against compute budget (FLOPs, GPU hours, training cost). Larger models require more compute at inference too, which is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides a detailed mapping from the CEFR framework's linguistic competence definition to specific benchmarks, justifying why each benchmark measures the claimed construct (e.g., WiC for lexical competence, BLiMP for grammatical competence)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; models are evaluated directly on benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage despite many benchmarks (BLiMP 2020, MNLI 2018, WiC 2019, TriviaQA 2017) predating all models by years."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information through context formatting or prompting."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples may overlap with training data across model families."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Internal factual knowledge scales significantly faster with model size than linguistic competence.",
       "evidence": "Linear regression: R²=0.81 for IFK vs R²=0.50 for linguistic competence; slope 0.059 vs 0.029 (Section 4.1.1, Figure 2). IFK shows 39.50% performance gap between large/small models vs 18.29% for linguistic competence (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "Linguistic competence stabilizes at moderate model sizes (5-7B parameters).",
       "evidence": "Mann-Whitney U tests show no significant improvement from medium to large models for linguistic competence (p=0.062, Table 2). Qwen2.5-3B scores 0.6909 vs OLMo-2-0325-32B at 0.7095 (Table 1).",
       "supported": "moderate"
     },
     {
       "claim": "External factual knowledge does not continue to improve with model size after a certain threshold.",
       "evidence": "Gemma-2-9b (9.24B) achieves highest EFK score (0.7961), outperforming 32B models. Mann-Whitney U test medium vs large p=0.591 (Table 2).",
       "supported": "moderate"
     },
     {
       "claim": "The FLM paradigm — smaller linguistically competent models with external factual retrieval — is viable.",
       "evidence": "Supported indirectly by the scaling analysis showing linguistic competence saturation. No direct demonstration of an FLM system with external retrieval.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "All benchmarks predate the models by years (BLiMP 2020, MNLI 2018, WiC 2019, TriviaQA 2017). Models may have trained on benchmark data, which would inflate scores — particularly for linguistic benchmarks, undermining the core finding that linguistic competence saturates early."
     },
     {
       "flag": "Architecture/training confounded with size",
       "detail": "Different model families (Qwen, Llama, OLMo, etc.) have different architectures and training data. The paper attributes differences to model size but cannot separate size from architecture and training effects. For example, Qwen2.5-3B outperforms 6-10B models from other families on linguistic tasks."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "Nine statistical tests are performed without Bonferroni or other corrections. At α=0.05 with 9 tests, the chance of at least one false positive is ~37%."
     },
     {
       "flag": "Single-run results with no variance",
       "detail": "All results appear to be single evaluations with no variance or confidence intervals reported, making it impossible to assess result stability."
     },
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature, sampling strategy, and other inference parameters are not reported for any of the 23 models, despite these settings affecting benchmark performance."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational work on neural scaling laws that this paper builds upon to argue for linguistic competence saturation."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2022,
       "relevance": "Benchmark used for internal factual knowledge evaluation; relevant to LLM evaluation methodology."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenges emergent abilities narrative; directly relevant to claims about scaling properties of LLM capabilities."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive nlp tasks",
       "authors": ["Patrick Lewis", "Ethan Perez"],
       "year": 2020,
       "relevance": "RAG foundational paper; the FLM paradigm proposes externalizing factual knowledge similarly to RAG."
     },
     {
       "title": "A framework for few-shot language model evaluation",
       "authors": ["Leo Gao", "Jonathan Tow"],
       "year": 2024,
       "relevance": "The LM Evaluation Harness used as the evaluation framework in this study."
     },
     {
       "title": "TinyStories: How small can language models be and still speak coherent english?",
       "authors": ["Ronen Eldan", "Yuanzhi Li"],
       "year": 2023,
       "arxiv_id": "2305.07759",
       "relevance": "Directly relevant to the FLM thesis — demonstrates small models can achieve linguistic coherence."
     },
     {
       "title": "A survey on evaluation of large language models",
       "authors": ["Yupeng Chang"],
       "year": 2024,
       "relevance": "Survey of LLM evaluation methods and benchmarks; relevant to evaluation methodology."
     },
     {
       "title": "ZebraLogic: On the scaling limits of LLMs for logical reasoning",
       "authors": ["Bill Yuchen Lin"],
       "year": 2025,
       "relevance": "Argues LLMs may have reached peak reasoning capabilities despite scale; supports the FLM thesis."
     }
   ]
 }