ai-research-survey

scan.json (31087B)

---

 {
   "paper": {
     "title": "Model Cascading for Code: A Cascaded Black-Box Multi-Model Framework for Cost-Efficient Code Completion with Self-Testing",
     "authors": [
       "Boyuan Chen",
       "Mingzhi Zhu",
       "Brendan Dolan-Gavitt",
       "Muhammad Shafique",
       "Siddharth Garg"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2405.15842",
     "doi": "10.48550/arXiv.2405.15842"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "A cascaded multi-model framework with self-testing reduces LLM code generation inference costs by an average of 26% (up to 70%) while maintaining or improving pass@1 accuracy compared to single-model self-testing. The approach uses self-generated test cases to decide when to escalate queries from smaller to larger models, with a learned threshold parameter. Over 90% of questions solved by the largest model in a family can also be solved by smaller models, validating the cascading premise. The optimal cascading threshold is typically high (0.9–1.0), and models below 10% accuracy should be excluded from the cascade.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper. The approach is described algorithmically but no implementation is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses three publicly available benchmarks: HumanEval, MBPP-sanitized, and APPS-test (introductory subset). All are standard public datasets referenced with citations."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 specifies NVIDIA GeForce RTX 3090 GPUs, CUDA Toolkit 11.8, PyTorch-based HuggingFace Transformers v4.31.0, and the Accelerate library. This is detailed enough to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. While the methodology is described in detail, there are no executable instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 3, and figures are reported as point estimates. No confidence intervals, error bars, or ± notation are provided for accuracy or cost savings figures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims cascading 'saves 70% of cost' and 'maintains or improves accuracy' based solely on comparing point estimates. No statistical significance tests (t-tests, bootstrap, etc.) are applied to any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost savings are reported with baseline context: e.g., Table 3 shows 17.4% savings on HumanEval for WizardCoder-Python, and figures show absolute accuracy and cost values for both cascading and single-model approaches, enabling the reader to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 30% validation / 70% test split is stated but not justified. No power analysis or explanation for why this ratio was chosen. The datasets have 164, 427, and 1000 questions respectively, but no discussion of whether these sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 notes 'Time is averaged across 10 runs' for cost statistics, but accuracy results appear to be single-run. No standard deviation, IQR, or spread measures are reported for the main accuracy or cost savings results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against two baselines: randomly selected single-model self-testing plans (Section 5.1) and Pareto-optimal single-model self-testing plans from the validation set (Section 5.2, purple crosses in Figure 7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The model families used (Codegen-mono, WizardCoder-V1.0, WizardCoder-Python-V1.0) are from 2022–2023. Newer model families like DeepSeek-Coder, Qwen-Coder, and Code Llama are cited in Section 2 but not evaluated. No comparison with other cost-reduction methods like FrugalGPT (also cited)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 ablates the threshold parameter θ (Figure 6). Section 5.2 compares cascading vs. single-model Pareto solutions. The parameter space exploration across k, l, and θ effectively ablates each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics are used throughout: pass@1 accuracy and inference cost ($/1k queries or $/1M tokens). Both are reported for all experiments."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via pass/fail on test suites. No human evaluation of code quality, readability, or correctness is conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes a 30% validation / 70% test split. Parameters (Pareto-optimal combinations) are selected on the validation set and evaluated on the held-out test set. Section 4.2 confirms 'we randomly sampled 30% of the questions for the validation set... and evaluated... on the remaining 70%.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-model, per-dataset accuracy. Table 3 shows cost savings broken down by model family and dataset. Figure 7 shows detailed results for each model family–dataset combination."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses the failure case of WizardCoder-V1.0 on APPS-Intro (-1.6% cost savings), explaining that the 1B model has too low accuracy, 'generates incorrect answers and tests, thereby increasing costs on wasted computations.' They recommend excluding models below 10% accuracy."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports -1.6% cost savings for WizardCoder-V1.0 on APPS-Intro, meaning the cascading scheme actually performed worse. This is discussed honestly in Section 5.1."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'reduces costs by an average of 26%, and up to 70%.' Table 3 shows individual savings ranging from -1.6% to 70.0%. Averaging the reported values yields approximately 24–26%. The 'up to 70%' claim is supported by the Codegen family on HumanEval. The claim about 'maintaining or improving accuracy' is supported by Figure 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims cascading 'reduces costs' — a causal claim. The experimental design (controlled comparison of cascading vs. single-model with the same models and datasets, Pareto-optimal selection on validation set) provides adequate support. The ablation on θ (Section 5.3) shows the mechanism."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims applicability to 'Code' generally, and the abstract says 'natural language generation tasks.' But all experiments are on Python-only benchmarks (HumanEval, MBPP, APPS) with three specific model families. The conclusion states 'the method is language-agnostic and can be applied to other programming languages' without evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations. The cost savings could be specific to the model families chosen (where larger models have large overlap with smaller models). The paper doesn't discuss whether the results hold for model families with different overlap characteristics or different architecture relationships."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@1 accuracy and inference cost, and frames its claims in terms of these same quantities. It does not overclaim that pass@1 represents general 'code quality' or 'developer productivity.' The claims match the granularity of the measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model names and sizes are specified: WizardCoder-Python-V1.0 (7B, 13B, 34B), WizardCoder-V1.0 (1B, 3B, 15B), Codegen-mono (350M, 2B, 6B, 16B). These are specific open-source model identifiers sufficient to download the exact models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the prompting approach (generate solutions, generate test cases with assert statements) but does not provide the actual prompt text used. Figure 4 shows an example problem but not the prompt template sent to the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: temperature 0.8 for sampling, temperature 0 for greedy search (Section 3.2), maximum 1024 tokens per prompt (Section 4.3), k ∈ {-1, 0, 1, 3, 5, 10}, l ∈ {0, 2, 4}, θ ∈ {0.0, 0.1, ..., 1.0}."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. The cascading pipeline is a sequential multi-model inference framework, not an agent with tool use, memory, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the datasets, the restriction to APPS introductory-level questions, the 30/70 validation/test split procedure, and the check that validation/test accuracy differences are within 5%. The filtering criteria for APPS (models below 10% accuracy excluded) are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. The conclusion (Section 6) mentions future work directions (per-model θ, speculative decoding) but does not discuss limitations of the current approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not consider threats such as the representativeness of the 30% validation split, sensitivity to the specific model families, or whether the cost model generalizes beyond RTX 3090 setups."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states 'our experiments focused on Python code generation tasks' but then claims 'the method is language-agnostic and can be applied to other programming languages' — extending rather than bounding scope. No explicit statement of what the results do NOT show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, test results, per-question cascading decisions) is released. Only aggregated results in tables and figures are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is well-described: standard benchmarks (HumanEval 164 questions, MBPP-sanitized 427 questions, APPS-Intro 1000 questions), model inference setup (Section 4.3), and cost measurement procedure (Section 3.4 with batching details)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: prompts go to models → solutions and tests generated → pairwise testing → scoring via ns × nt → threshold comparison → cascade or accept (Section 3.1, Figures 3–4). The validation/test split process is also described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: NYU Tandon School of Engineering and NYU Abu Dhabi eBRAIN Lab. The authors evaluate open-source models, not products from their own institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot be assessed because no funding source is disclosed. Absence of a funding statement prevents evaluation of funder independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the three model families (Codegen-mono, WizardCoder-V1.0, WizardCoder-Python-V1.0). The training datasets are named (THEPILE, BIGQUERY, BIGPYTHON, Code Alpaca) but no temporal boundaries are given."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval, MBPP, or APPS questions appeared in the training data of any model. The models were released after these benchmarks were published, making contamination plausible."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021), MBPP (2021), and APPS (2021) were all published before the models were trained. No contamination analysis or discussion is provided despite the clear temporal risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on code benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost is a primary focus. Table 1 shows $/1M tokens for all models. Table 2 details time and cost per 1M tokens with GPU counts and batch sizes. Section 3.4 explains the cost calculation methodology using RTX 3090 rental rates ($0.44/hr)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 and Table 2 detail the GPU configuration: number of RTX 3090 GPUs per model, inference time in hours, batch sizes. The rental cost basis is specified ($0.44/hr per GPU from RunPod)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of sensitivity to random seeds. The validation/test split is random, and sampling uses temperature 0.8, but no multi-seed results are reported for accuracy."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 states 'Time is averaged across 10 runs' for cost measurements, but the main accuracy results do not state how many runs produced them. The critical pass@1 results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The full parameter search space is explicitly defined: k ∈ {-1, 0, 1, 3, 5, 10}, l ∈ {0, 2, 4}, θ ∈ {0.0, 0.1, ..., 1.0} for each model in the family. All valid combinations are enumerated and evaluated on the validation set (Section 3.2)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Configurations are selected via Pareto optimality on the validation set (Section 3.2, Figure 5). The selection criterion is clearly described: points where no other combination achieves both higher accuracy and lower cost."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical hypothesis tests are performed in this paper. Comparisons are made via point estimates without formal testing, so correction for multiple comparisons is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both the cascading system and the single-model baselines. No discussion of potential bias from self-implementation of baselines. No independent evaluation or acknowledgment of this limitation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "This is the core contribution. All results (Figures 1, 7) plot accuracy as a function of cost. The entire framework is designed to find Pareto-optimal cost–accuracy tradeoffs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval, MBPP, and APPS adequately measure 'code completion' ability as claimed. The paper treats these benchmarks as ground truth without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The cascading framework is applied uniformly across all model comparisons within each family. Single-model baselines use the same self-testing scoring mechanism (CodeT). The comparison isolates the cascading strategy rather than confounding it with different scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval, MBPP, and APPS were all published in 2021, before the evaluated models were trained. No discussion of whether model training data includes benchmark solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The self-testing scheme generates tests from the model itself, which could introduce information feedback, but this is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test data overlap or share structural similarities. Models within each family share training data, but the implications for independence are not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines."
       }
     }
   },
   "claims": [
     {
       "claim": "The cascading approach reduces inference costs by an average of 26%, and up to 70%, while maintaining or improving accuracy compared to single-model self-testing.",
       "evidence": "Table 3 shows per-family, per-dataset cost savings: 17.4% to 70.0% (with one negative case at -1.6%). Figure 1 and Figure 7 show Pareto-optimal cascading solutions consistently below or equal to single-model solutions on cost-accuracy plots.",
       "supported": "moderate"
     },
     {
       "claim": "Over 90% of questions solved by the largest model in a family can also be solved by smaller models.",
       "evidence": "Figure 2 Venn diagram for WizardCoder-Python-V1.0: of 131 questions solvable by 34B, 119 (91%) are also solvable by 7B or 13B. Section 1 states 'over 90% of the questions solved by the 34B model can also be solved by one of the smaller models.'",
       "supported": "strong"
     },
     {
       "claim": "The optimal cascading threshold θ is typically high (0.9 or 1.0), reflecting the high accuracy of self-generated tests.",
       "evidence": "Section 5.3 reports optimal θ values are 'typically high, either 1.0 or 0.9.' Figure 6 shows highest accuracy at θ = 0.8 for one configuration. Less capable families exhibit lower optimal θ on MBPP.",
       "supported": "moderate"
     },
     {
       "claim": "Models below 10% accuracy should be excluded from cascading, as they waste computation on incorrect answers and tests.",
       "evidence": "Section 5.1 discusses WizardCoder-V1.0 on APPS-Intro where the 1B model (3.9% accuracy) 'generates incorrect answers and tests, thereby increasing costs on wasted computations.' Table 3 shows -1.6% savings for this case.",
       "supported": "weak"
     },
     {
       "claim": "The approach is fully black-box and compatible with any model family.",
       "evidence": "Section 1 states the method 'requires no access to internal model parameters.' Experiments demonstrate it across three different model families (Codegen-mono, WizardCoder-V1.0, WizardCoder-Python-V1.0).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Outdated model families",
       "detail": "All three model families (Codegen-mono, WizardCoder-V1.0, WizardCoder-Python-V1.0) are from 2022–2023 and significantly behind the state of the art. Newer code LLMs like DeepSeek-Coder, Qwen-Coder, and Code Llama are cited but not evaluated. It is unclear whether the high overlap between model sizes (a key premise) holds for newer families."
     },
     {
       "flag": "No statistical testing on main claims",
       "detail": "All cost savings percentages (e.g., '26% average', '70% best case') are computed from point estimates without any significance tests, confidence intervals, or variance measures. The random validation/test split introduces variance that is not quantified."
     },
     {
       "flag": "No contamination analysis",
       "detail": "HumanEval, MBPP, and APPS were all published in 2021, before the model families were trained. The paper makes no effort to assess whether benchmark solutions appear in training data, which could affect both absolute accuracy numbers and the cascading threshold."
     },
     {
       "flag": "Cost model assumptions may not generalize",
       "detail": "The cost model is based on RTX 3090 GPU rental rates from RunPod (2023 pricing), batch inference, and excludes input prompt costs. Real deployment scenarios with different hardware, pricing, and serving configurations could yield different cost-accuracy tradeoffs."
     },
     {
       "flag": "No code released",
       "detail": "Despite describing a detailed algorithmic framework, no source code is released, making it difficult to verify the results or apply the approach to new model families."
     }
   ],
   "cited_papers": [
     {
       "title": "Deepseek-coder: When the large language model meets programming – the rise of code intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang", "Zhenda Xie"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "State-of-the-art code generation LLM family relevant to benchmarking and model cascading approaches."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code LLM family with multiple model sizes, directly applicable to cascading approaches."
     },
     {
       "title": "CodeT: Code generation with generated tests",
       "authors": ["Bei Chen", "Fengji Zhang", "Anh Nguyen", "Daoguang Zan", "Zeqi Lin"],
       "year": 2023,
       "doi": "10.18653/v1/2024.emnlp-main.1118",
       "relevance": "Foundational work on self-testing for code generation that this paper builds upon for cascading threshold decisions."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Key reference for LLM self-improvement through self-generated feedback, motivating the self-testing approach."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Prior work on LLM model cascading for natural language tasks; closest related work to this paper's approach."
     },
     {
       "title": "Large language model cascades with mixture of thought representations for cost-efficient reasoning",
       "authors": ["Murong Yue", "Jie Zhao", "Min Zhang", "Liang Du", "Ziyu Yao"],
       "year": 2024,
       "relevance": "Related work on LLM cascading for cost reduction in reasoning tasks."
     },
     {
       "title": "WizardCoder: Empowering code large language models with evol-instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Primary model family used in the experiments; two WizardCoder families are evaluated."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark used as a primary evaluation dataset in this paper."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used as a primary evaluation dataset in this paper."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Introduces APPS benchmark used as an evaluation dataset in this paper."
     },
     {
       "title": "A survey on model compression for large language models",
       "authors": ["Xunyu Zhu", "Jian Li", "Yong Liu", "Can Ma", "Weiping Wang"],
       "year": 2024,
       "doi": "10.1162/tacl_a_00704",
       "relevance": "Surveys alternative approaches to reducing LLM inference costs (quantization, pruning, distillation) that this paper positions against."
     },
     {
       "title": "Fast inference from transformers via speculative decoding",
       "authors": ["Yaniv Leviathan", "Matan Kalman", "Yossi Matias"],
       "year": 2023,
       "relevance": "Alternative inference acceleration technique discussed as complementary to model cascading."
     },
     {
       "title": "Qwen2.5-coder technical report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Recent code LLM family that could be used for cascading but was not evaluated."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners running multi-model code completion servers could adopt the cascading approach, but the framework requires validation-set optimization and multiple models deployed simultaneously."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Cascading from small to large models is intuitive; the main contribution is formalizing the threshold-based approach with self-testing rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications; purely an efficiency optimization."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy; straightforward cost optimization research."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code released, no demo, no tool available to try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "NYU is a recognized institution but the work does not involve high-profile products or labs."
     }
   }
 }