ai-research-survey

scan.json (30376B)

---

 {
   "paper": {
     "title": "LaCy: What Small Language Models Can and Should Learn is Not Just a Question of Loss",
     "authors": [
       "Szilvia Ujváry",
       "Louis Béthune",
       "Pierre Ablin",
       "João Monteiro",
       "Marco Cuturi",
       "Michael Kirchhof"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.12005"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LaCy combines spaCy grammar parsing with loss signals to decide which tokens an SLM should learn vs delegate to a larger model via a <CALL> token. On a 334M parameter model cascading with Llama 3.2 1B, LaCy achieves 22.71% FactScore vs 15.89% baseline and the lowest fact leakage (11.28%). The key insight is that cross-entropy loss is blind to token type: high-loss non-factual tokens are often acceptable alternatives, while high-loss factual tokens cause hallucinations. The method's spaCy annotation runs on CPU only, making it cheaper than GPU-based alternatives like LLM judge annotations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dwiki dataset is publicly available from the OLMo2 project (Groeneveld et al., 2024). The LLM judge annotations are available at 'kilian-group/LMLM-pretrain-dwiki6.1M' (Section A.2.2). FactScore and NLU benchmarks are standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 and 4 describe architecture and hyperparameters, and Section A.4 mentions 8 A100-80GB GPUs and bfloat16 precision. However, there is no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts for replicating experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Figures 2, 6, 7; Tables 1, 6) report point estimates only with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'LaCy outperforms all previous methods' (Section 5.2) are based solely on comparing point estimates without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 reports 'an increase of 6.88% compared to the baseline' for FactScore. Figures 2 and 11-12 show absolute values for all methods, providing context for the magnitude of improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes: the acceptability experiment uses 112 documents (~44k tokens), FactScore uses 183 entities, and NLU uses standard benchmarks. No power analysis or sample size reasoning is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experiment. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes four baselines: vanilla Baseline (no CALL), Loss-based calls, LLM judge (Zhao et al., 2025), and Rho-1 (Lin et al., 2024). All are reimplemented with the same budgets and data."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Rho-1 (Lin et al., 2024) and LLM judge (Zhao et al., 2025), which are recent and represent the state of the art in token selection for pretraining."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 presents five ablations: spaCy only, spaCy + Reference Model, LaCy + Ignorefacts, LaCy + Ignore. Figure 6 and Table 5 show the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on FactScore (factual precision), Fact Leakage (factual QA without calling), NLU benchmarks (ARC-Easy, HellaSwag, PIQA, SIQA), and validation losses (call/non-call/total)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated: FactScore uses GPT-3.5 turbo, NLU uses log-likelihood comparison, and the acceptability analysis uses Gemini 2.0 Flash as judge. No human evaluation of generated outputs is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section A.4.3 states '10% of the dwiki dataset as validation set.' FactScore uses a fixed set of 183 entities (Section B). NLU benchmarks use their standard test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down NLU results by benchmark (ARC-Easy, HellaSwag, PIQA, SIQA). Figure 5 separates call vs non-call losses. Fact leakage is broken down by benchmark (BigBench QA Wikidata, PopQA long-tail)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 acknowledges 'the model sometimes tries to predict factual tokens it should not.' Figure 4 shows example generations with factual errors highlighted in red. Section 5.2 notes 'not all facts inserted by the cascade partner are true.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 shows LaCy+Ignore degrades NLU performance. Section 5.7 shows loss is not correlated with FactScore. Figure 10 shows ablation improvements disappear when equalizing forward steps. Section 5.5 reports that freeing capacity by offloading facts does not improve NLU."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims higher FactScores and outperformance of Rho and LLM-judge SLMs. Figure 2 confirms LaCy achieves 22.71% vs Rho-1 21.63% and LLM judge 20.97%. The claim of being 'simpler and cheaper' is supported by Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('LaCy models successfully learn which tokens to predict') are supported by controlled experiments: all methods use the same architecture, data, and compute budget, varying only the token selection mechanism. Ablations in Section 5.4 isolate the contribution of each component."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'What Small Language Models Can and Should Learn' is broad, but all experiments are on Wikipedia domain only (dwiki dataset), with GPT-2 architectures at 334M and 1.3B scale. Section 6 calls it 'an explorative pilot study' but the title and framing do not bound claims to this narrow setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for why LaCy outperforms baselines. Section 5.4 ablates components but does not consider whether improvements could be due to confounds such as the specific grammar parser, the dwiki domain, or the particular cascade model."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes between its measurements (FactScore, fact leakage, NLU benchmarks) and their interpretation. Section 5.7 explicitly warns that validation loss is not a valid proxy for factual accuracy in token-selection settings. The paper does not overclaim from its proxies."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "GPT-2 architectures are trained from scratch with full specifications in Table 3. Cascade models are specified as 'Llama-3.2-1B' and 'Qwen 3 32B' with references to model cards and technical reports. Gemini 2.0 Flash (used only for acceptability analysis) and GPT-3.5 turbo (for FactScore) are also named."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full acceptability judgment prompt is provided in Section A.1. RAG prompt for Qwen 3 32B is provided in Section A.5. FactScore prompt template is given in Section B: 'Tell me a bio of <name>. <name> is'. PopQA rephrasing approach is described."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports batch size, total steps, learning rate, warmup, and precision for all model sizes. Section A.4 specifies AdamW with weight decay 0.1, gradient accumulation across 4 steps, context length 1024. Section A.5 specifies greedy decoding with repetition penalty 1.2."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The cascade mechanism (SLM generates until CALL token, then cascade model generates one token) is simple and fully described in Section A.5, but this is not scaffolding in the agentic sense."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section A.2.1 provides detailed documentation of the spaCy annotation pipeline: named entity processing, supplementary entity detection with specific heuristics, grammatical word classification, and tokenization. Section A.2.2 documents the LMLM data processing conversion."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Conclusion) contains two substantive paragraphs discussing limitations: small-scale training may cause the model to predict factual tokens it shouldn't, and the paper excludes the question of what to do after calling. While not a separate section, it is substantive."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 identifies specific threats: (1) 'The model sometimes tries to predict factual tokens it should not, which we believe is mostly because it was trained at a small scale,' and (2) the exclusion of the downstream handling of CALL tokens. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 6 calls this 'an explorative pilot study' and Section 1 states 'we focus fully on the first question,' the paper does not explicitly state what its results do NOT show — e.g., it doesn't bound claims to the Wikipedia domain, GPT-2 architectures, or the specific model scales tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The dwiki dataset is public, but experimental outputs (model generations, FactScore evaluations, spaCy annotations, trained model weights) are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section A.2 describes the dwiki dataset source (OLMo2 project, ~3B tokens of Wikipedia). The acceptability experiment uses 112 validation documents covering ~44k tokens (Section 3). Data processing steps are documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public datasets (dwiki, FactScore entities, NLU benchmarks)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section A.2.1 documents the full annotation pipeline from raw text to token-level labels in detailed steps (named entity processing → supplementary detection → grammatical classification → tokenization). Section A.2.2 documents the LMLM conversion pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure. Section 7 (Acknowledgements) thanks individuals but does not mention any funding sources or grants. Apple is listed as primary affiliation but no explicit funding statement."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: 'Apple, †University of Cambridge, work done as an intern at Apple.' The Apple trademark notice is also included."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Apple has a commercial interest in SLM technology (Apple Intelligence, on-device models). The research was conducted at Apple, and Apple benefits from demonstrating that SLMs can be effective with appropriate training methods. This conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. Apple employees researching SLM effectiveness for a company that deploys SLMs commercially represents an undisclosed potential financial interest."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The SLMs are trained from scratch on dwiki, so their training data is known. However, the cascade model Llama 3.2 1B (whose outputs directly affect FactScore) has no training cutoff stated. FactScore uses GPT-3.5 turbo with no cutoff mentioned either."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between dwiki training data and evaluation benchmarks (FactScore entities, NLU test sets). The SLM is trained on Wikipedia and evaluated on biography generation from Wikipedia — the overlap is obvious but unaddressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "NLU benchmarks (ARC-Easy, HellaSwag, PIQA, SIQA) and QA benchmarks (BigBench, PopQA) were all published before the models' training. No contamination analysis is provided for either the SLM or the cascade partner Llama 3.2 1B."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports preprocessing overhead for label generation, but inference cost/latency for the cascade system (SLM + Llama 3.2 1B) is not reported. The added cost of cascading (querying a larger model ~22% of tokens) is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section A.4: 'train most of our models on 8 A100-80GB GPUs' and 'Training on 8 GPUs finishes in 3 days.' Table 2 reports preprocessing overhead (152 h/1B tokens on CPU for LaCy). Table 4 specifies total training steps."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The learning rate, batch size, and other settings appear chosen without documented search. The CALL percentage (15%) is matched to the LLM judge baseline for fairness but not searched."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The 15% call rate is explicitly justified as matching the LLM judge baseline for fair comparison (Section 4). Training steps are calibrated to equalize gradient signals (Section A.4). Hyperparameters follow standard practices from prior work."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 5 methods, 6+ metrics, and 5 ablation variants. No statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare LaCy against their own reimplementations of all baselines (Loss-based, Rho-1, LLM judge). Section 5.1 notes they 'reimplement and pretrain these methods' but does not acknowledge the potential bias of evaluating their own system against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and Figure 10 compare performance when equalizing training steps. Table 2 reports preprocessing overhead. Section A.4 explains that calling models train 15% longer to equalize the number of tokens receiving gradient signals."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "FactScore is used to measure factual precision. Section 5.7 questions loss as a metric but does not question whether FactScore adequately captures the claimed capability of factual accuracy in generation. FactScore relies on GPT-3.5 turbo for fact verification, which itself can err."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All methods use the same cascade setup with Llama 3.2 1B, the same call budget (22% of tokens), and the same decoding strategy. Section A.5 describes the cascade protocol applied uniformly. The scaffold is controlled across comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the dwiki training data contains information that could leak into benchmarks temporally. The SLM is trained on Wikipedia which contains factual information also tested by FactScore."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The FactScore evaluation uses Wikipedia pages as ground truth, and the model is trained on Wikipedia data (dwiki) — the feature overlap between training and evaluation is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training (dwiki, a Wikipedia subset from OLMo2) and evaluation (FactScore biography generation verified against Wikipedia) draw from the same source (Wikipedia). This obvious non-independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No checks for overlap between dwiki training documents and FactScore evaluation entities."
       }
     }
   },
   "claims": [
     {
       "claim": "LaCy achieves the highest FactScore (22.71%) among all methods when cascading with Llama 3.2 1B, a 6.88 percentage point improvement over the no-call baseline (15.89%).",
       "evidence": "Figure 2 (left) and Section 5.2. LaCy 22.71%, Rho-1 21.63%, LLM judge 20.97%, Loss-based 19.44%, Baseline 15.89%. Single-run results on 183 FactScore entities.",
       "supported": "moderate"
     },
     {
       "claim": "LaCy achieves the lowest fact leakage (11.28%) on factual QA benchmarks, meaning the least factual knowledge was trained into the SLM's parametric memory.",
       "evidence": "Figure 2 (right). LaCy 11.28%, LLM judge 14.25%, Rho-1 15.91%, Baseline 18.45%, Loss-based 19.55%. Evaluated on BigBench QA Wikidata and PopQA long-tail.",
       "supported": "moderate"
     },
     {
       "claim": "Cross-entropy loss is blind to the type of error: non-factual tokens with high loss are often acceptable alternative continuations, while high-loss factual tokens are not.",
       "evidence": "Section 3 and Figure 3 (right). Acceptability experiment on a single batch of 112 documents (~44k tokens) with Gemini 2.0 Flash as judge, showing non-facts have higher acceptability than facts at the same loss levels.",
       "supported": "moderate"
     },
     {
       "claim": "Factual offloading does not significantly degrade Natural Language Understanding.",
       "evidence": "Table 1. LaCy achieves 39.9% average NLU vs 39.6% baseline across ARC-Easy, HellaSwag, PIQA, SIQA. Differences are less than 2 percentage points across all methods.",
       "supported": "moderate"
     },
     {
       "claim": "Validation loss is not correlated with FactScore in the token-selection setting.",
       "evidence": "Section 5.7 and Figure 7. Neither call loss, non-call loss, nor total loss is predictive of FactScore across the compared methods.",
       "supported": "weak"
     },
     {
       "claim": "LaCy's spaCy-based labeling has minimal throughput overhead and runs on CPU, not GPU.",
       "evidence": "Table 2. LaCy: 152 h/1B tokens on a single CPU core. LLM judge: 233 h/1B tokens on a single A100 GPU. Rho-1: 56 h/1B tokens on a single A100 GPU.",
       "supported": "strong"
     },
     {
       "claim": "The loss signal is necessary: spaCy annotation alone (without loss thresholding) performs worse than LaCy.",
       "evidence": "Section 5.4 and Figure 6. spaCy only achieves lower FactScore and higher fact leakage than full LaCy.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across all experiments (FactScore, fact leakage, NLU, validation losses, ablations) are single-point estimates with no confidence intervals, standard deviations, or indication of variance across runs. It is impossible to assess whether the differences between methods are statistically meaningful."
     },
     {
       "flag": "Apparent single-run experiments",
       "detail": "No mention of multiple random seeds or repeated runs anywhere in the paper. Given that the differences between methods are modest (e.g., LaCy 22.71% vs Rho-1 21.63% on FactScore, a 1.08pp gap), a single run provides no evidence that these differences are robust."
     },
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "All Apple-affiliated authors research SLM methods at a company that commercially deploys SLMs (Apple Intelligence). The paper promotes SLM capability through delegation, which aligns with Apple's product strategy. No funding disclosure or competing interests statement is present."
     },
     {
       "flag": "Training-evaluation domain overlap",
       "detail": "The SLM is trained on dwiki (Wikipedia data) and evaluated on FactScore (biography generation verified against Wikipedia). This domain overlap is never discussed, raising concerns about whether the approach generalizes beyond Wikipedia-style content."
     },
     {
       "flag": "Self-reimplemented baselines",
       "detail": "All baselines (Loss-based, Rho-1, LLM judge) are reimplemented by the authors (Section 5.1). While using the same budgets and data is fair, the authors' implementations of competing methods may not match the original authors' best configurations, per Lucic et al. (2018)."
     }
   ],
   "cited_papers": [
     {
       "title": "Physics of language models: part 3.1, knowledge storage and extraction",
       "authors": ["Zeyuan Allen-Zhu", "Yuanzhi Li"],
       "year": 2024,
       "relevance": "Foundational work on LLM knowledge storage capacity limits, which motivates the token delegation approach."
     },
     {
       "title": "Small language models are the future of agentic AI",
       "authors": ["Peter Belcak", "Greg Heinrich", "Shizhe Diao", "Yonggan Fu", "Xin Dong", "Saurav Muralidharan", "Yingyan Celine Lin", "Pavlo Molchanov"],
       "year": 2025,
       "arxiv_id": "2506.02153",
       "relevance": "Argues for SLMs as the future of agentic AI, directly relevant to why SLM capability and delegation matter."
     },
     {
       "title": "Prioritized training on points that are learnable, worth learning, and not yet learnt",
       "authors": ["Sören Mindermann", "Jan M Brauner", "Muhammed T Razzak"],
       "year": 2022,
       "relevance": "Introduces Rho-loss for prioritized training, a key baseline and predecessor to the token selection approach."
     },
     {
       "title": "Not all tokens are what you need for pretraining",
       "authors": ["Zhenghao Lin", "Zhibin Gou", "Yeyun Gong"],
       "year": 2024,
       "relevance": "Proposes Rho-1, a key baseline that selects tokens based on loss difference with a reference model."
     },
     {
       "title": "Pre-training large memory language models with internal and external knowledge",
       "authors": ["Linxi Zhao", "Sofian Zalouk", "Christian K. Belardi"],
       "year": 2025,
       "arxiv_id": "2505.15962",
       "relevance": "Key baseline (LMLM/LLM judge) that uses GPT-4o to annotate factual tokens for delegation during pretraining."
     },
     {
       "title": "I don't know: Explicit modeling of uncertainty with an [IDK] token",
       "authors": ["Roi Cohen", "Konstantin Dobler", "Eden Biran", "Gerard de Melo"],
       "year": 2024,
       "relevance": "Proposes shifting logits onto an IDK token when models are wrong, related to the CALL token delegation approach."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessi"],
       "year": 2023,
       "relevance": "Foundational work on teaching LMs to use external tools, the broader paradigm that LaCy's CALL mechanism participates in."
     },
     {
       "title": "FActScore: Fine-grained atomic evaluation of factual precision in long form text generation",
       "authors": ["Sewon Min", "Kalpesh Krishna", "Xinxi Lyu"],
       "year": 2023,
       "doi": "10.18653/v1/2023.emnlp-main.741",
       "relevance": "The primary evaluation metric used in this paper for measuring factual accuracy of generated text."
     },
     {
       "title": "How much do language models memorize?",
       "authors": ["John X. Morris", "Chawin Sitawarin", "Chuan Guo"],
       "year": 2025,
       "arxiv_id": "2505.24832",
       "relevance": "Studies LLM memorization limits as a function of parameters, directly motivating the capacity argument for SLM delegation."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws work; the paper argues that findings linking loss with downstream performance do not transfer to the token-selection setting."
     },
     {
       "title": "Learning to route LLMs with confidence tokens",
       "authors": ["Yu-Neng Chuang", "Prathusha Kameswara Sarma", "Parikshit Gopalan"],
       "year": 2025,
       "relevance": "Proposes analyzing which tokens a trained model is wrong on for routing, related to the token selection problem."
     },
     {
       "title": "Self-RAG: Learning to retrieve, generate, and critique through self-reflection",
       "authors": ["Akari Asai", "Zeqiu Wu", "Yizhong Wang"],
       "year": 2024,
       "relevance": "Self-reflective retrieval-augmented generation, related to the model cascade and delegation approach."
     }
   ]
 }