ai-research-survey

scan.json (25991B)

---

 {
   "paper": {
     "title": "Poodle: Seamlessly Scaling Down Large Language Models with Just-in-Time Model Replacement",
     "authors": [
       "Nils Strassenburg",
       "Boris Glavic",
       "Tilmann Rabl"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.05525",
     "doi": "10.48550/arXiv.2512.05525"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "Poodle presents a just-in-time model replacement (JITR) vision where recurring LLM tasks are automatically detected and replaced with fine-tuned surrogate models. On IMDB sentiment classification, replacing Llama 405B Turbo with a fine-tuned BERT model achieves 82× cost reduction for 1M requests, 19.6× throughput improvement, and competitive accuracy (0.90 vs 0.937). Model search followed by fine-tuning on 500 samples completes 4× faster than training from scratch on 5,000 samples while reaching higher accuracy (0.91 vs 0.89).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided. Poodle is described as a prototype but no source code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The experiments use the publicly available IMDB movie reviews dataset [18] (Maas et al., 2011)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided. The paper mentions using an NVIDIA RTX A5000 GPU but no software environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The experimental setup is described in prose but lacks sufficient detail for exact replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Table 2 shows single accuracy values and Figures 7-8 show point estimates without uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims like 'model search followed by fine-tuning outperforms other approaches' (Section 4.4) are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context: '60× cheaper' and '82× cheaper' for cost (Section 4.1), '19.6× more items per second' for throughput (Section 4.2), and accuracy differences with baselines (e.g., BERT 0.90 vs LLM 0.937 in Table 2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the sample sizes used. The choice of 10,000 IMDB items, 500/5,000 training samples, and 10 models for search is not explained."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across any experiment. All results are single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 compares four approaches: baseline (BERT fine-tuned on 5,000 items), S-naive (fine-tune all 10 models), S-500 (model search + 500 samples), S-5000 (model search + 5,000 samples). Cost comparisons use three LLM baselines (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The main model used for accuracy/inference experiments is Llama-2-7B (2023), which is outdated by 2025 standards. The cost analysis uses GPT-4.1 and Llama 405B Turbo pricing (contemporary), but the actual experimental model is old. BERT (2018) as surrogate is appropriate for the task but the LLM baseline is weak."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 systematically varies components: with/without model search, and different training set sizes (500 vs 5,000), showing the contribution of each component (Figure 8)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates across three metrics: monetary cost (Section 4.1), inference time/throughput (Section 4.2), and accuracy (Section 4.3)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to the claims, which are about automated cost, latency, and accuracy metrics on a classification task."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states '10,000 random items from the IMDB dataset with a 50/50 training-test split,' clearly separating training and test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by LLM model (GPT-4.1 nano, GPT-4.1, Llama 405B Turbo in Figure 7), by training data size (500-5,000 in Table 2), by label source (ground truth vs LLM-generated), and by approach (Figure 8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases are discussed. The paper does not show where JITR breaks down, what types of tasks it fails on, or scenarios where the surrogate model performs poorly."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every experiment shows improvement from JITR. No approaches that were tried and failed are mentioned."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'significant savings for exemplary tasks,' which is supported by the cost (82×), time (19.6×), and accuracy results. The hedging ('exemplary tasks') appropriately bounds the claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 claims model search 'outperforms' other approaches. The comparison is adequately controlled: same dataset, same task, same evaluation, single-variable manipulation (with/without model search, different data sizes)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Seamlessly Scaling Down Large Language Models' broadly, but all experiments use a single task (IMDB sentiment classification). The abstract uses 'exemplary tasks' (plural) when only one task was tested. The vision in Section 2 discusses general recurring tasks far beyond what was demonstrated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For example, IMDB sentiment is known to be a solved task for fine-tuned models — the paper does not consider whether results would hold for harder tasks. The LLM contamination on IMDB is mentioned but not analyzed as an alternative explanation for the accuracy comparison."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures accuracy on IMDB sentiment classification and frames it as evidence that JITR works for general 'recurring tasks.' The gap between this single-task proxy and the broader claim of general JITR effectiveness is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The accuracy experiment specifies 'NousResearch/Llama-2-7b-chat-hf' [20] and 'bert-base-uncased' [13], but cost experiments reference marketing names ('GPT-4.1', 'GPT-4.1-nano', 'Llama 405B Turbo') without API versions or snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided: Figure 3 shows the base sentiment classification prompt, Figure 4 shows the wrapper prompt, and Figure 5 shows the expected response format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports epochs (2-9) and Section 4.2 mentions batch sizes (16 for Llama, 128 for BERT), but learning rate, optimizer, temperature, and other critical parameters are not reported. Section 4.3 notes 'we do not tune hyperparameters' but does not state the defaults used."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Poodle is a pipeline system (data collector → model search → fine-tuning → monitoring), not an agentic scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '10,000 random items from the IMDB dataset' and mentions BERT uses 256 tokens vs LLM 1024 tokens, but does not document tokenization, truncation strategy, or other preprocessing steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. Section 5 ('Summary and Discussion') discusses future work directions and challenges but does not explicitly address limitations of the current evaluation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address concerns like single-dataset evaluation, contamination effects on accuracy comparisons, or generalizability limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not say what the results do NOT show. The word 'preliminary' appears once, and 'exemplary' in the abstract, but there is no explicit scoping of claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The IMDB dataset is publicly available, but the experimental results (model search rankings, fine-tuning curves, cost calculations) are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes '10,000 random items from the IMDB dataset with a 50/50 training-test split.' Section 4.4 describes selecting 10 models from HuggingFace with specific categories (base BERT, 3 sentiment, 6 non-sentiment). Data source and selection process are clear."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is the standard IMDB benchmark dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The general JITR pipeline is described architecturally (Figure 2, Section 3), but the specific experimental data pipeline — how IMDB items were selected, how LLM labels were generated, how models were evaluated — lacks step-by-step documentation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. No acknowledgments section is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Hasso Plattner Institute (Uni Potsdam) and University of Illinois Chicago. Academic affiliations with no apparent conflict with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of funding disclosure is not the same as absence of funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama-2-7B for accuracy experiments but does not state its training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly notes 'the LLM might have seen the IMDB data before' as a caveat when interpreting the accuracy comparison between the LLM and BERT surrogate."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 acknowledges contamination risk: 'the LLM might have seen the IMDB data before.' The IMDB dataset (2011) predates Llama-2's training period. The paper notes this as a factor favoring the LLM in the comparison."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 provides detailed cost analysis with token prices (Table 1), break-even points, and per-1M-request savings ($33 for GPT-4.1 nano, $850 for GPT-4.1, $1420 for Llama 405B Turbo). Model development cost estimated at $4."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states '$4, which roughly equals three hours of an AWS A10G GPU instance' for surrogate model development. Section 4.2 specifies the NVIDIA RTX A5000 GPU used for inference experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented as single values without indication of repetition."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly states 'we do not tune hyperparameters,' transparently reporting zero hyperparameter search budget. Section 4.4 reports the model search budget: 10 models evaluated, 500 samples used for search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 describes using Alsatian's baseline approach with 500 samples to rank models, selecting the highest-ranked model. The ground truth ranking is generated by fully fine-tuning all 10 models, and model search correctly identifies the best."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed in the paper, so correction for multiple comparisons is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (Poodle) against baselines they implemented without acknowledging author-evaluation bias. No independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 and Section 4.4 explicitly show development time vs. accuracy for all four approaches, allowing direct comparison of compute cost and resulting performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses IMDB sentiment classification as a proxy for 'recurring tasks' generally, but does not discuss whether this well-studied, simple benchmark is representative of the diverse recurring tasks JITR targets."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the model comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "IMDB (2011) predates Llama-2's training data. The paper mentions 'the LLM might have seen the IMDB data before' but does not formally address temporal leakage or its effect on the accuracy comparison."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (e.g., prompt format, wrapper prompt) leaks information that affects the comparison."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test splits beyond stating '50/50 training-test split.' No analysis of potential overlap or structural similarity."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are applied. Contamination risk is only mentioned in passing."
       }
     }
   },
   "claims": [
     {
       "claim": "JITR with Poodle achieves 60-82× cost reduction for 1M requests compared to using LLMs directly.",
       "evidence": "Section 4.1, Figure 7: Poodle saves $850 (60×) vs GPT-4.1 and $1,420 (82×) vs Llama 405B Turbo for 1M sentiment classification requests, after a break-even point of <10,000 requests for large models.",
       "supported": "moderate"
     },
     {
       "claim": "JITR reduces inference time by 7.5-10× compared to using a small LLM (Llama-2-7B).",
       "evidence": "Section 4.2: BERT processes 19.6× more items per second than Llama-2-7B. For 1M requests, Poodle takes 7.5× less time; for 2M requests, more than 10×.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuned BERT models achieve competitive accuracy with LLMs on sentiment classification.",
       "evidence": "Section 4.3, Table 2: BERT fine-tuned on 5,000 LLM-generated labels reaches 0.90 accuracy vs LLM's 0.937, using only 256 tokens vs 1024 and no hyperparameter tuning.",
       "supported": "moderate"
     },
     {
       "claim": "Model search followed by fine-tuning outperforms alternative model development approaches in time, accuracy, and data efficiency.",
       "evidence": "Section 4.4, Figure 8: S-500 (model search + 500 samples) completes 4× faster than baseline and 19× faster than naive search while achieving 0.91 accuracy (vs 0.89 baseline). Model search correctly ranks sentiment-specific models highest.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-dataset evaluation",
       "detail": "All experiments use only IMDB sentiment classification — one of the simplest and most well-studied NLP tasks. No evidence that JITR works for other recurring task types (e.g., NER, summarization, translation) despite broad claims."
     },
     {
       "flag": "No error bars or variance",
       "detail": "All experimental results are single-run point estimates with no variance reporting. The accuracy differences (e.g., 0.89 vs 0.91) may not be statistically significant."
     },
     {
       "flag": "Claims outrun evidence",
       "detail": "The paper's vision and title ('Seamlessly Scaling Down Large Language Models') imply broad generality, but the prototype is only tested on binary sentiment classification. The gap between the vision (Section 2) and evaluation (Section 4) is substantial."
     },
     {
       "flag": "Contamination confound in accuracy comparison",
       "detail": "The LLM (Llama-2-7B) was likely trained on IMDB data, inflating its accuracy. The paper acknowledges this but does not quantify the effect, making the accuracy comparison between LLM and BERT surrogate difficult to interpret."
     },
     {
       "flag": "Missing failure analysis",
       "detail": "No failure cases or limitations of the approach are empirically demonstrated. The paper does not show tasks where JITR fails or where surrogate models underperform, despite this being a prototype evaluation."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Directly relevant work on LLM cost reduction through model cascading, addressing the same problem of reducing inference cost."
     },
     {
       "title": "Distilling the Knowledge in a Neural Network",
       "authors": ["Geoffrey Hinton", "Oriol Vinyals", "Jeff Dean"],
       "year": 2015,
       "arxiv_id": "1503.02531",
       "relevance": "Foundational knowledge distillation technique used by Poodle to transfer LLM knowledge into smaller surrogate models."
     },
     {
       "title": "HuggingGPT: Solving AI Tasks with ChatGPT and Its Friends in Hugging Face",
       "authors": ["Yongliang Shen", "Kaitao Song", "Xu Tan", "Dongsheng Li", "Weiming Lu", "Yueting Zhuang"],
       "year": 2023,
       "relevance": "Agentic approach to task-solving via model selection on HuggingFace, related to the model search component of JITR."
     },
     {
       "title": "Large Language Models as Tool Makers",
       "authors": ["Tianle Cai", "Xuezhi Wang", "Tengyu Ma", "Xinyun Chen", "Denny Zhou"],
       "year": 2024,
       "relevance": "Alternative approach to LLM cost reduction by generating code tools for tasks, compared against JITR in the introduction."
     },
     {
       "title": "CREATOR: Tool Creation for Disentangling Abstract and Concrete Reasoning of Large Language Models",
       "authors": ["Cheng Qian", "Chi Han", "Yi Fung", "Yujia Qin", "Zhiyuan Liu", "Heng Ji"],
       "year": 2023,
       "relevance": "LLM tool creation approach for task automation, related alternative to model replacement for reducing LLM dependency."
     },
     {
       "title": "Knowledge Distillation in Automated Annotation: Supervised Text Classification with LLM-Generated Training Labels",
       "authors": ["Nicholas Pangakis", "Samuel Wolken"],
       "year": 2024,
       "arxiv_id": "2406.17633",
       "relevance": "Directly evaluates using LLM-generated labels to train smaller models, the core training approach used by JITR."
     },
     {
       "title": "Fine-tuned small LLMs (still) significantly outperform zero-shot generative AI models in text classification",
       "authors": ["Martin Juan José Bucher", "Marco Martini"],
       "year": 2024,
       "arxiv_id": "2406.08660",
       "relevance": "Evidence that fine-tuned small models outperform LLMs on text classification, supporting JITR's core premise."
     },
     {
       "title": "Alsatian: Optimizing Model Search for Deep Transfer Learning",
       "authors": ["Nils Strassenburg", "Boris Glavic", "Tilmann Rabl"],
       "year": 2025,
       "doi": "10.1145/3725264",
       "relevance": "The model search system underlying Poodle's model selection component, from the same authors."
     },
     {
       "title": "Which Model to Transfer? Finding the Needle in the Growing Haystack",
       "authors": ["Cedric Renggli", "André Susano Pinto", "Luka Rimanic"],
       "year": 2022,
       "relevance": "Model search technique for transfer learning that addresses the same scaling challenge Poodle faces in finding surrogate models."
     }
   ]
 }