ai-research-survey

scan.json (28895B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "Interfaze: The Future of AI is built on Task-Specific Small Models",
     "authors": [
       "Harsha Vardhan Khurdula",
       "Vineet Agarwal",
       "Yoeven D Khemlani"
     ],
     "year": 2026,
     "venue": "IEEE Conference on Artificial Intelligence (CAI) 2026",
     "arxiv_id": "2602.04101"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository or URL is provided anywhere in the paper. The system architecture is described but no code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation benchmarks are publicly available standard datasets: MMLU-Pro, MMLU, GPQA-Diamond, AIME-2025, LiveCodeBench v5, MMMU, AI2D, ChartQA, Common Voice v16. References with URLs are provided for each."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details are provided. The paper mentions 'separate GPU pools with batching and caching' (Section III-B) but gives no specifics."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are included. The paper describes architecture conceptually but provides no runnable scripts, commands, or step-by-step instructions to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table II reports only point estimates (e.g., '83.6', '91.38') with no confidence intervals, error bars, or uncertainty measures on any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements like '+3.0 (MMLU-Pro)', '+55.3 (AIME-2025)' over GPT-4.1 based solely on comparing two numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V reports point differences with baseline context (e.g., '+3.0 (MMLU-Pro), +1.18 (MMLU), +55.3 (AIME-2025)') and Table II shows both system scores and baseline scores, providing enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of sample sizes for any benchmark evaluation. The number of test examples is not stated for any benchmark, nor is there any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multiple-run results reported. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table II compares against seven baselines: GPT-4.1, GPT-5 (Minimal Reasoning), Claude Sonnet 4, Gemini 2.5 Flash, Claude Sonnet 4 (Thinking), Claude Opus 4 (Thinking), and Gemini 2.5 Pro."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include very recent models: GPT-5, Claude Opus 4, Gemini 2.5 Pro, Claude Sonnet 4 — all contemporary frontier models at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section V mentions ablation observations: 'removing OCR/diagram/chart parsers drops AI2D/ChartQA by 4–7 points; disabling context compilation costs ≈2 points on GPQA-Diamond; turning off the optional short reasoning head hurts AIME and MMLU-Pro in high-difficulty slices.' However, no ablation table is provided — these are mentioned only in passing."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Nine benchmarks are used spanning knowledge, reasoning, code, multimodal, and speech: MMLU-Pro, MMLU, GPQA-Diamond, AIME-2025, LiveCodeBench v5, MMMU, AI2D, ChartQA, Common Voice v16."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is performed. All evaluation is automated benchmark scoring."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard benchmark test sets (MMLU, GPQA-Diamond, AIME-2025, LiveCodeBench v5, MMMU validation split, AI2D, ChartQA, Common Voice v16) which have established train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table II provides per-benchmark breakdown across nine different benchmarks. Section V-A provides per-domain discussion (knowledge, science, math, coding, multimodal, speech)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No specific failure cases or error analyses are shown. Section VI discusses engineering limitations (delay, over-building) but does not show examples where the system fails or produces incorrect outputs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges that LiveCodeBench v5 (57.77) trails Sonnet 4 Thinking by 7.73 and Gemini 2.5 Pro by 18.13, and explicitly attributes this to 'a deliberate choice to favor lightweight sandbox checks over longer agentic debugging loops.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All specific benchmark numbers claimed in the abstract (83.6% MMLU-Pro, 91.4% MMLU, 81.3% GPQA-Diamond, 57.8% LiveCodeBench v5, 90.0% AIME-2025, 77.3% MMMU, 91.5% AI2D, 90.9% ChartQA, 90.8% Common Voice) match Table II."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'most of the improvement comes from the small-model and tool stack and the way it compiles context' (Section V). While ablations are mentioned in passing, they are not presented in a controlled table and the comparison is between a tooled system and non-tooled baselines, making it impossible to isolate the causal contribution of each component rigorously."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'The Future of AI is built on Task-Specific Small Models' — an extremely broad generalization from benchmark evaluations on nine specific tasks. The paper does not bound its claims to the tested benchmarks or acknowledge that results may not generalize to other domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper does not consider whether gains come from tool access (search, code sandbox) rather than the small-model architecture, or whether baselines with equivalent tool access would close the gap."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures benchmark accuracy scores and frames them as evidence that 'the future of AI is built on task-specific small models' — a claim about production deployment economics and architecture. The gap between benchmark performance and the claimed practical superiority of the architecture is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Baselines are listed as 'GPT-4.1', 'GPT-5 (Minimal Reasoning)', 'Claude Sonnet 4', etc. — marketing names without snapshot dates or API versions. The paper's own small models are described only at the architecture level ('a stack of convolutional and self-attention blocks') with no model names or versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No prompts or system instructions are provided for any benchmark evaluation. The paper describes that prompts are compiled by the context layer but does not show actual prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported — no temperature, top-p, max tokens, learning rates, or sampling settings for any model in the system."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in substantial detail across Sections III-IV: the ingress stage, small-model perception stack, context construction layer with schema (observations, entities, relations, provenance), action layer with controller and tool chains, and fallback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the perception pipeline is described architecturally (OCR, ASR, context compilation), no details are provided about how benchmark inputs were specifically prepared, preprocessed, or formatted for evaluation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Limitations and Future Work' discusses delay from SLM fan-out and cold starts, and over-building of context when the controller invokes more tools than needed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VI identifies specific threats: 'Delay stems from context fan out from SLMs (OCR/ASR, scraping, retrieval) and the bounded agentic loop, plus cold starts for small models' and 'Over-building happens when the controller invokes more tools or retrieval passes than are needed for a given query, inflating cost without clear quality gains.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state specific scope boundaries. The title claims 'The Future of AI' but the limitations section discusses only engineering improvements (delay, over-building), not what domains, modalities, or use cases the results do NOT apply to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw evaluation data (per-example predictions, model outputs, logs) is available. Only aggregate accuracy numbers are reported in Table II."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states which benchmarks were used but does not describe how evaluations were conducted — which splits were used, how inputs were formatted, or how scoring was performed."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; all evaluation uses standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The architecture pipeline is described but the evaluation data pipeline is not. No details on how benchmark inputs flow through the system during evaluation, or how scoring is performed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. All three authors are affiliated with JigsawStack, Inc. but no funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list 'JigsawStack, Inc.' as their affiliation with email addresses at jigsawstack.com. The affiliation with the company behind the product is clear."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are employees of JigsawStack, Inc., the company behind the Interfaze product. The employer has a direct commercial interest in positive benchmark results for their system."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included, despite all authors being employees of the company whose product is being evaluated."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training cutoff dates are stated for any model — not for the proprietary small models trained in-house, nor for the final LLM, nor for the baseline models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap for any model or benchmark. The small models are 'trained on a mixture of public and proprietary data' (Section III-B) with no overlap analysis."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No contamination analysis despite using well-known public benchmarks (MMLU published 2020, GPQA 2023, etc.) that could appear in training data of both the small models and the final LLM."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper argues for cost efficiency ('shifting the bulk of computation away from the most expensive and monolithic models') but reports no actual costs, latencies, token counts, or per-example timing."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. No GPU hours, API costs, hardware specs, or training time for the small models."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple seed experiments. All results appear to be from single runs with no sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from single runs or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported despite the system having many tunable components (controller thresholds, context budgets, confidence thresholds for OCR fallback, etc.)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the system configuration was selected. The controller is 'trained on offline tuples of requests, tool chains, and pass/fail labels' (Section III-E) with no detail on configuration selection."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "JigsawStack employees evaluate their own Interfaze product against third-party models without acknowledging author-evaluation bias. No independent evaluation or bias mitigation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims cost efficiency but does not report performance as a function of compute. No comparison at matched compute budgets with baselines."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmarks measure what the paper claims. The paper uses AIME-2025 and MMLU to support claims about 'the future of AI' without discussing construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Interfaze-Beta is compared against standalone LLMs (GPT-4.1, Claude, Gemini) without equivalent tool access. The comparison conflates the tool stack contribution with the architecture. The paper partially acknowledges this ('most of the improvement comes from the small-model and tool stack') but does not test baselines with equivalent tool augmentation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The small models are 'trained on a mixture of public and proprietary data' with no temporal analysis of whether benchmark data appears in training sets."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. The tool stack includes web search and retrieval, which could leak benchmark answers during evaluation. No discussion of whether the search/retrieval tools could access benchmark solutions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test independence for any model in the system."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, decontamination, or temporal splits."
       }
     }
   },
   "claims": [
     {
       "claim": "Interfaze-Beta achieves 83.6% on MMLU-Pro, 91.4% on MMLU, 81.3% on GPQA-Diamond, 57.8% on LiveCodeBench v5, 90.0% on AIME-2025, 77.3% on MMMU, 91.5% on AI2D, 90.9% on ChartQA, and 90.8% on Common Voice v16.",
       "evidence": "Table II provides head-to-head benchmark results with seven baselines. Numbers match abstract claims.",
       "supported": "moderate"
     },
     {
       "claim": "Interfaze-Beta improves over GPT-4.1 by a macro-average of +13.53 points (median +5.61) on shared benchmarks.",
       "evidence": "Section V computes differences from Table II across seven shared benchmarks. However, the comparison is between a tool-augmented system and a standalone LLM, making the comparison inherently unfair.",
       "supported": "weak"
     },
     {
       "claim": "Most queries are handled primarily by the small-model and tool stack, with the large LLM operating only on distilled context.",
       "evidence": "Described architecturally in Sections III-IV. No quantitative data on what fraction of compute or queries are handled by small models vs. the final LLM. The claim is an architectural assertion rather than an empirical finding.",
       "supported": "weak"
     },
     {
       "claim": "Removing OCR/diagram/chart parsers drops AI2D/ChartQA by 4-7 points; disabling context compilation costs ~2 points on GPQA-Diamond.",
       "evidence": "Mentioned in passing in Section V with no ablation table, no details on experimental setup, and no error bars. These numbers are stated without supporting methodology.",
       "supported": "weak"
     },
     {
       "claim": "The system achieves competitive accuracy while shifting the bulk of computation away from the most expensive models.",
       "evidence": "No cost data, latency measurements, or compute breakdowns are provided. The claim about compute shifting is entirely unsupported by empirical evidence.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Interfaze-Beta, a tool-augmented system combining small task-specific DNNs/SLMs for perception (OCR, ASR, object detection) with context construction and a final LLM, reports competitive scores across nine benchmarks including 90.0% on AIME-2025 and 91.4% on MMLU. The paper's central architectural claim — that small models building structured context for a final LLM can match or beat standalone frontier models — is supported by benchmark scores but undermined by the unfair comparison (tooled system vs. non-tooled baselines), absence of cost data, and lack of statistical rigor. Ablation results are mentioned in passing but not presented in a rigorous table.",
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "All three authors are JigsawStack employees evaluating Interfaze, a JigsawStack product. No independent evaluation, no competing interests statement, and no acknowledgment of this conflict."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "Interfaze-Beta (with web search, code sandbox, OCR, ASR, retrieval tools) is compared against standalone LLMs without equivalent tool access. The improvement is attributed to the architecture, but adding equivalent tools to baselines might close the gap. The paper partially acknowledges this but does not test tool-augmented baselines."
     },
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All nine benchmark results are single point estimates with no confidence intervals, standard deviations, or multi-run results. The AIME-2025 score of 90.0% in particular (a small test set) could have high variance."
     },
     {
       "flag": "Claims far exceed evidence",
       "detail": "The title claims 'The Future of AI is built on Task-Specific Small Models' based on benchmark results from one system on nine tasks. No production deployment data, cost analysis, or generalizability evidence supports this broad claim."
     },
     {
       "flag": "Incomplete baseline table",
       "detail": "Table II has many blank cells (dashes) — 5 of 8 baselines are missing ChartQA, AI2D, and Common Voice results. The paper may appear stronger in these areas simply because baselines are not reported."
     },
     {
       "flag": "Ablation without rigor",
       "detail": "Ablation results are mentioned in a single paragraph without a table, without error bars, and without describing the ablation methodology. These unverifiable numbers could be cherry-picked."
     },
     {
       "flag": "Search/retrieval tool may leak benchmark answers",
       "detail": "The system includes web search and retrieval tools. On benchmarks like MMLU and GPQA, the system could retrieve answers from the web during evaluation. This is not discussed or controlled for."
     },
     {
       "flag": "Proprietary training data with no details",
       "detail": "Small models are 'trained on a mixture of public and proprietary data' (Section III-B) with no further detail. The proprietary data could contain benchmark-related content, and there is no contamination analysis."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Cost-aware LLM cascading — directly relevant to model routing and cost-efficiency methodology."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["D. Ding", "A. Mallick", "C. Wang"],
       "year": 2024,
       "relevance": "LLM routing between small local and large cloud models — core topic of the survey's routing coverage."
     },
     {
       "title": "Universal Model Routing for Efficient LLM Inference",
       "authors": ["W. Jitkrittum"],
       "year": 2025,
       "arxiv_id": "2502.08773",
       "relevance": "Generalized LLM routing across expert pools — relevant to efficient inference methodology."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["T. Schick", "J. Dwivedi-Yu", "R. Dessì"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Foundational work on tool-augmented LLMs, directly relevant to agentic AI capabilities."
     },
     {
       "title": "HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in Hugging Face",
       "authors": ["Y. Shen", "K. Song", "X. Tan"],
       "year": 2023,
       "arxiv_id": "2303.17580",
       "relevance": "LLM as planner over specialist model registry — key precedent for multi-model orchestration systems."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["S. Yao", "J. Zhao", "D. Yu"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agentic prompting pattern interleaving reasoning and tool use."
     },
     {
       "title": "Chameleon: Plug-and-Play Compositional Reasoning with Large Language Models",
       "authors": ["P. Lu", "B. Peng", "H. Cheng"],
       "year": 2023,
       "arxiv_id": "2304.09842",
       "relevance": "Composable model+tool pipelines — relevant to evaluation of multi-component LLM systems."
     },
     {
       "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark",
       "authors": ["Y. Wang", "X. Ma", "G. Zhang"],
       "year": 2024,
       "arxiv_id": "2406.01574",
       "relevance": "Key benchmark for evaluating LLM knowledge and reasoning capability."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["N. Jain", "K. Han", "A. Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Code generation benchmark designed to address contamination — directly relevant to code evaluation methodology."
     },
     {
       "title": "LLM With Tools: A Survey",
       "authors": ["Z. Shen"],
       "year": 2024,
       "arxiv_id": "2409.18807",
       "relevance": "Survey of tool-augmented LLM design patterns and failure modes."
     },
     {
       "title": "Small Language Models are the Future of Agentic AI",
       "authors": ["P. Belcak", "G. Heinrich", "S. Diao"],
       "year": 2025,
       "arxiv_id": "2506.02153",
       "relevance": "Position paper on small models for agentic workflows — directly relevant to the small model vs. large model debate."
     },
     {
       "title": "Small Models are Valuable Plug-ins for Large Language Models",
       "authors": ["C. Xu", "Y. Xu", "S. Wang"],
       "year": 2024,
       "relevance": "Small models as specialist components in LLM systems — relevant to model composition methodology."
     },
     {
       "title": "Unified Scaling Laws for Routed Language Models",
       "authors": ["A. Clark", "D. de las Casas", "A. Guy"],
       "year": 2022,
       "relevance": "Scaling laws for mixture-of-experts models — relevant to routing and model efficiency research."
     }
   ]
 }