ai-research-survey

scan-v5.json (26284B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "LaCy: What Small Language Models Can and Should Learn is Not Just a Question of Loss",
     "authors": [
       "Szilvia Ujváry",
       "Louis Béthune",
       "Pierre Ablin",
       "João Monteiro",
       "Marco Cuturi",
       "Michael Kirchhof"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.12005",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims (loss is predictive but insufficient, spaCy parser helps, LaCy achieves higher FactScores, outperforms Rho/LLM-judge) are substantiated by experiments in Sections 3, 5.2, and 5.3.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about LaCy improving FactScore are supported by controlled pretraining experiments with multiple baselines reimplemented on identical budgets and data, plus ablations in Section 5.4.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Experiments are confined to a single Wikipedia domain (dwiki) with GPT-2 architectures up to 1.3B, yet the paper title and framing suggest broad applicability to SLMs generally; the authors acknowledge in the conclusion that 'this study is an explorative pilot study.'",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why LaCy improves FactScore—e.g., whether gains stem from call calibration, training token equalization artifacts, or the specific cascade partner rather than the token selection itself.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes between FactScore (factual precision in biography generation), fact leakage (parametric memorization measured via QA benchmarks with calling disabled), and NLU performance—each measuring a distinct aspect.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section; the only limitation discussion is two sentences in the conclusion noting it is a 'pilot study' and that behavior 'is mostly because it was trained at a small scale.'",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are enumerated; the paper does not discuss confounds such as domain specificity, the impact of cascade model quality, or the small acceptability experiment (112 documents).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it addresses only 'when to call,' not 'what to do once an SLM calls for help,' and Section 2.3 devotes a subsection to explaining why the latter is out of scope.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement or grant acknowledgment appears anywhere in the paper; the acknowledgments only name individuals for helpful conversations.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed on the title page: Apple and University of Cambridge (intern at Apple).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The majority of authors are Apple employees and the work was conducted at Apple; the results (more efficient SLMs using external calls) directly align with Apple's commercial interest in on-device AI.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations appear in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: SLMs (around or below 1B parameters), acceptability (factual/logical consistency with ground truth, not exact match), FactScore, model cascade, and <CALL> token mechanism are all precisely defined in context.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly contributes LaCy, a pretraining method that combines spaCy factuality annotations with loss signals to determine <CALL> token placement, and frames it as improving over purely loss-based or LLM-judge approaches.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides detailed engagement with Rho-1/Rho-loss, LLM judge methods, IDK token approaches, and cascade literature, explicitly explaining how LaCy refines each prior approach rather than just listing citations.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository URL is provided; the paper mentions relying on a codebase 'kickstarted' by Awni Hannun but does not release it.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The dwiki dataset is from the publicly available OLMo2 project; LLM judge annotations are cited as available at kilian-group/LMLM-pretrain-dwiki6.1M; evaluation benchmarks (BigBench, PopQA, FactScore entities) are standard public datasets.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Training hardware (8 A100-80GB GPUs), precision (bfloat16/float32), and framework choices are described, but no requirements file, Dockerfile, or complete software environment specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided; the appendix describes the methods but not a reproducible pipeline someone could follow without guessing implementation details.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All main results (Figures 2, 5, 6) are reported as single point estimates with no error bars, confidence intervals, or standard deviations across runs.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to any comparative claims; differences between methods are presented without testing whether they exceed chance variation.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute improvements are reported—e.g., LaCy achieves 22.71% vs. baseline 15.89% FactScore (+6.82pp), and 11.28% vs. 18.45% fact leakage—providing interpretable effect magnitudes relative to baseline.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The acceptability experiment uses 112 documents (one validation batch) without justification; the 183 FactScore biography entities are inherited from the benchmark without power analysis.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No standard deviation, variance, or results across multiple training runs are reported; each method appears to have been trained once.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Four baselines are included and reimplemented with identical budgets: no-<CALL> baseline, loss-based calls, LLM judge (Zhao et al. 2025), and Rho-1 (Lin et al. 2024).",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All baselines are recent (2022–2025) and directly comparable; they are reimplemented from scratch rather than copied from prior papers, ensuring fair comparison.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.4 ablates four LaCy variants: spaCy only, spaCy+Reference Model, LaCy+Ignorefacts, and LaCy+Ignore, systematically isolating contributions of the loss and factuality components.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation uses FactScore (factual precision), fact leakage (QA containment), NLU benchmarks (ARC-Easy, HellaSwag, PIQA, SIQA), and validation loss breakdowns (call/non-call/total).",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "No human evaluation is conducted; acceptability is judged by Gemini 2.0 Flash acting as an LLM judge, not human raters.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "10% of dwiki is reserved as validation set (Section A.4.3); FactScore is evaluated on 183 biography entities from the held-out benchmark.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by model scale (334M vs. 1.3B), by metric type (FactScore vs. fact leakage vs. NLU), and by individual NLU benchmark in Tables 1 and 6.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.2 notes 'the model sometimes tries to predict factual tokens it should not' and Figure 4 explicitly shows examples of non-factual retrievals (e.g., retrieved token 'the' in wrong context).",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.7 reports that validation loss is uncorrelated with FactScore, and Section C.2/Figure 10 shows LaCy+Ignore is not beneficial once training steps are equalized—both reported as negative findings.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact model versions are specified: Llama 3.2 1B (Meta AI, 2024), Qwen 3 32B (Qwen Team, 2025), Gemini 2.0 Flash; GPT-2 architectures described by parameter count, dimension, heads, and layers in Table 3.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "The full acceptability judging prompt with examples is provided in Appendix A.1; the RAG prompt for Qwen 3 32B is provided in Appendix A.5; the FactScore biography prompt is described in Appendix B.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 reports batch size, total training steps, learning rate, warmup steps, and precision for all model variants; AdamW with weight decay 0.1 and no LR scheduling beyond warmup is noted.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The cascade inference mechanism is described in detail in Appendix A.5, including tokenizer mismatch handling, the running-quantile <CALL> logit calibration, and the RAG cascade setup.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.2.1 provides a step-by-step description of the spaCy annotation pipeline including named entity processing, supplementary entity detection, grammatical word classification, and tokenization mapping.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "dwiki is from OLMo2 (public); LLM judge annotations are cited as available at kilian-group/LMLM-pretrain-dwiki6.1M; evaluation benchmarks are standard public resources.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section A.2 describes data sourcing (dwiki from OLMo2, 3B Wikipedia tokens), LLM judge annotation processing, and the spaCy annotation pipeline in sufficient detail.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; all data is from public datasets and existing model annotations.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.2.1–A.2.2 documents the full pipeline from raw Wikipedia tokens through spaCy annotation, tokenization, and conversion of LLM judge labels to the <CALL> format used in training.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Neither the training data cutoff of Llama 3.2 1B (used as cascade partner) nor of Gemini 2.0 Flash (used as acceptability judge) is stated or discussed.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The model is pretrained on Wikipedia (dwiki) and evaluated on FactScore (Wikipedia biography entities); potential overlap between training documents and evaluation entities is not acknowledged or addressed.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The cascade partner Llama 3.2 1B was almost certainly pretrained on Wikipedia data that overlaps with FactScore entities; this contamination of the cascade retrieval is not discussed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 reports preprocessing overhead per 1B tokens (e.g., LaCy: 152h on single CPU core vs. LLM judge: 233h on A100 GPU); cascade inference cost is qualitatively discussed in terms of call rate (~22%).",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Section A.4.1 states training completes in 3 days on 8 A100-80GB GPUs; Table 4 provides exact training steps for all model variants enabling compute estimation.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Cross-entropy loss is blind to token type: non-factual tokens with high loss are often acceptable continuations and should not trigger a <CALL>.",
       "evidence": "Figure 3 shows that factual tokens have lower acceptability at high loss quantiles while non-factual high-loss tokens remain largely acceptable, validated via Gemini 2.0 Flash acceptability scoring on 112 documents.",
       "supported": "moderate"
     },
     {
       "claim": "LaCy outperforms all baseline methods on FactScore for biography generation (22.71% vs. 21.63% for Rho-1).",
       "evidence": "Figure 2 (left) shows LaCy at 22.71% vs. Rho-1 at 21.63%, LLM judge at 20.97%, loss-based at 19.44%, baseline at 15.89% for 334M model with Llama 3.2 1B cascade.",
       "supported": "moderate"
     },
     {
       "claim": "LaCy achieves the lowest fact leakage (11.28%) compared to all baselines, confirming minimal parametric memorization of factual tokens.",
       "evidence": "Figure 2 (right) shows LaCy fact leakage of 11.28% vs. LLM judge 14.25%, Rho-1 15.91%, loss-based 19.55%, baseline 18.45% on BigBench QA Wikidata and PopQA.",
       "supported": "moderate"
     },
     {
       "claim": "Validation loss is not correlated with FactScore in the token-selection setting, unlike standard pretraining.",
       "evidence": "Figure 7 shows no monotonic relationship between call loss, non-call loss, or total loss and FactScore across the five compared methods, contradicting scaling law predictions.",
       "supported": "strong"
     },
     {
       "claim": "LaCy's spaCy-based preprocessing costs only CPU compute (152h/1B tokens on a single CPU core) versus GPU compute for LLM judge (233h on A100).",
       "evidence": "Table 2 directly compares preprocessing overhead per 1B tokens for all methods; LaCy is the only GPU-free labeling approach other than the trivial loss-based baseline.",
       "supported": "strong"
     },
     {
       "claim": "Factual offloading does not significantly degrade NLU performance; LaCy matches or slightly exceeds the baseline on NLU benchmarks.",
       "evidence": "Table 1 shows LaCy at 39.9% average NLU vs. baseline 39.6%, and LLM judge at 39.1%, with differences within ~1pp across ARC-Easy, HellaSwag, PIQA, SIQA.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "LaCy is a pretraining method for Small Language Models that combines spaCy grammar-based factuality detection with the model's own loss signal to identify which tokens should be delegated via a <CALL> token to a cascade model rather than learned. The core insight is that loss alone cannot distinguish factual errors from acceptable alternative continuations: high-loss non-factual tokens are often semantically valid, while high-loss factual tokens are genuinely unlearnable and dangerous. LaCy achieves higher FactScores than loss-based, Rho-1, and LLM-judge baselines while simultaneously minimizing parametric memorization of facts, and it does so with CPU-only preprocessing overhead. Crucially, the paper demonstrates that validation loss is decoupled from factual accuracy in token-selection settings, undermining the standard practice of using loss as a proxy for pretraining quality.",
   "red_flags": [
     {
       "flag": "Single domain, single dataset",
       "detail": "All experiments use only the Wikipedia dwiki corpus; generalization to other domains is entirely untested, yet claims are framed broadly for SLMs in general."
     },
     {
       "flag": "No confidence intervals or repeated runs",
       "detail": "All results are single point estimates from one training run per method; without variance across seeds or runs, observed differences (e.g., 22.71% vs. 21.63% FactScore) cannot be assessed for reliability."
     },
     {
       "flag": "Train/test contamination unaddressed",
       "detail": "Models are pretrained on Wikipedia (dwiki) and evaluated on FactScore (Wikipedia biography entities); the cascade partner Llama 3.2 1B also saw Wikipedia data during pretraining—none of this overlap is acknowledged."
     },
     {
       "flag": "Acceptability experiment extremely small",
       "detail": "The key motivating experiment (Section 3, Figure 3) uses only 112 documents (one validation batch) scored by an LLM judge, with no quantitative uncertainty reported."
     },
     {
       "flag": "Self-described as pilot study",
       "detail": "Authors explicitly call this 'an explorative pilot study' in the conclusion, but the framing throughout the paper and abstract presents findings as general contributions without appropriate hedging."
     },
     {
       "flag": "No code release",
       "detail": "Despite claiming reproducibility benefits and describing the pipeline in detail, no code is released, making independent replication difficult."
     },
     {
       "flag": "Undisclosed funding from interested party",
       "detail": "Work performed at Apple with no funding disclosure; Apple has direct commercial interest in efficient SLMs for on-device use."
     }
   ],
   "cited_papers": [
     {
       "title": "Not all tokens are what you need for pretraining (Rho-1)",
       "relevance": "Direct baseline and prior work on token selection for pretraining efficiency"
     },
     {
       "title": "FActScore: Fine-grained atomic evaluation of factual precision in long form text generation",
       "relevance": "Primary evaluation metric used throughout the paper"
     },
     {
       "title": "Pre-training large memory language models with internal and external knowledge (LMLM/LLM judge)",
       "relevance": "Key baseline providing LLM judge annotations and the dwiki dataset"
     },
     {
       "title": "Language model cascades: Token-level uncertainty and beyond",
       "relevance": "Foundation for the cascade inference framework used in experiments"
     },
     {
       "title": "Prioritized training on points that are learnable, worth learning, and not yet learnt (Rho-loss)",
       "relevance": "Conceptual predecessor to loss-based token selection methods"
     },
     {
       "title": "Physics of language models: part 3.1, knowledge storage and extraction",
       "relevance": "Motivates the claim that LLM storage is limited by parameter count"
     },
     {
       "title": "Scaling laws for neural language models",
       "relevance": "Establishes the loss-performance correlation that LaCy shows breaks down in token-selection settings"
     },
     {
       "title": "Cascade-aware training of language models",
       "relevance": "Prior work on training models aware of cascade inference, used as loss-based calls baseline"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "SLM deployment with function calling is practically relevant to on-device AI, but the method requires full pretraining from scratch, limiting immediate adoption."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that validation loss is decoupled from factual accuracy in token-selection settings directly contradicts the scaling law consensus that guides most pretraining decisions."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns raised; the paper is narrowly focused on factual accuracy in biography generation."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Challenges the 'train on all tokens' orthodoxy of foundation model training but does so without confrontational framing."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Biography generation examples are shown in Figure 4 but no interactive demo or released model is available for practitioners to try."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Apple affiliation and use of Llama, Gemini, and Qwen models lend brand recognition, though the paper itself trains small custom GPT-2 architectures."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }