ai-research-survey

scan.json (28835B)

---

 {
   "paper": {
     "title": "RooflineBench: A Benchmarking Framework for On-Device LLMs via Roofline Analysis",
     "authors": [
       "Zhen Bi",
       "Xueshu Chen",
       "Luoyang Sun",
       "Yuhang Yao",
       "Qing Shen",
       "Jungang Lou",
       "Cheng Deng"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.11506"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C states: 'The code for this work is available at https://github.com/banbu-ai/roofline_bench.' A concrete GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Raw benchmark measurement data (inference logs, memory traces) is not released as a downloadable dataset. The paper shows results in figures and tables but no raw data files are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware specifications are detailed in Table 1, and backends (CUDA, MPS) are mentioned. However, no requirements.txt, Dockerfile, or detailed software environment listing library versions is provided. Mentioning PyTorch and llama-bench without version pinning is insufficient."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the GitHub repository is linked and Appendix C describes experimental settings (models, precisions, scenarios), there are no step-by-step reproduction instructions (README with commands, scripts to replicate experiments)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. All results are point estimates — single TPS values in tables and single data points on Roofline plots."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'MLA consistently achieves the highest operational intensity and attainable performance') based solely on comparing numbers, with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as absolute values (GFLOPS, TPS) without standardized effect sizes. Comparative claims lack effect size quantification — no percentage improvements or relative magnitudes are systematically reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why this particular set of models, hardware platforms, or sequence length configurations was chosen. The number of repeated measurements per benchmark run is not stated."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Tables 3-5 show single TPS values. Run-to-run variability in inference benchmarks can be significant but is not addressed."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple models serve as comparisons: Qwen2.5, Llama 3.2, PLM, SmolLM2, Fox-1, Qwen3, and Pythia family. Different attention architectures (MHA, GQA, MLA) are compared head-to-head."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models tested are contemporary: Qwen2.5 (2024), Llama 3.2 (2024), PLM (2025), Qwen3 (2025), SmolLM2 (2024). Hardware includes recent platforms (M1 Pro, RTX 3090, Jetson Orin Nano Super)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies individual factors: model depth (2-64 layers, Sec 4.2), numerical precision (FP16/Q8_0/Q4_K_M, Sec 4.3.1), and attention mechanism (MHA/GQA/MLA, Sec 4.3.2), isolating each factor's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: Operational Intensity (FLOPs/Byte), Performance (GFLOPS/sec), Relative Inference Potential (Φ), and throughput (TPS for both prefilling and decoding)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems performance profiling paper measuring hardware utilization metrics. Human evaluation is structurally irrelevant to claims about inference throughput and operational intensity."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems benchmarking paper measuring inference performance, not a learning task with train/test splits. The concept of held-out test sets does not apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are extensively broken down by model architecture, hardware platform (5 devices), precision level (3 formats), sequence scenario (SISO/SILO/LISO/LILO), and layer count. Figures 2-6 and Tables 3-5 provide granular breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies the OI regression at increased depth (Insight 2), the 'efficiency trap' from hardware heterogeneity (Insight 5), GQA performing worst among the three architectures at 1.5B scale, and the severe memory-bound regime in SILO scenarios."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: OI regression beyond 3-5 layers despite increasing performance (Sec 4.2), GQA exhibiting the lowest efficiency among three attention mechanisms (Sec 4.3.2), and quantization providing diminishing returns in compute-heavy scenarios (Sec 4.3.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: (1) OI varies with sequence length (Fig 2, Sec 4.1), (2) OI regression with depth (Fig 3, Sec 4.2), (3) efficiency trap from hardware heterogeneity (Fig 5, Sec 5.1), (4) MLA unlocks inference potential (Fig 6, Sec 4.3.2)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('sequence length determines OI', 'MLA effectively compresses KV cache traffic') are supported by controlled single-variable experiments. Layer scaling holds architecture constant, attention comparisons use matched ~1.5B parameter counts. The study design adequately supports the causal direction."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally scoped to the tested configurations. The paper titles itself 'On-Device LLMs' and tests across 5 representative platforms. Future Work (Appendix B) explicitly acknowledges the need to expand to MoE architectures, more hardware, and different inference engines."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents its interpretations of the results (memory wall, KV cache compression) but does not systematically consider alternative explanations. For example, the OI regression could be influenced by software framework overhead rather than purely physical constraints, but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It measures FLOPs, throughput, and OI, and claims are about inference performance and hardware utilization — no proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'Qwen2.5-1.5B-Instruct', 'Llama-3.2-1B-Instruct', 'PLM-1.8B-Instruct', 'SmolLM2-1.7B-Instruct', 'Fox-1-1.6B', 'Qwen3-0.6B'. For open-weight models, the model name with size is the version."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting. It runs inference benchmarks via llama-bench with parameterized sequence lengths, not prompt-based evaluation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key parameters are reported: precisions (FP16, Q8_0, Q4_K_M), sequence length configurations (SISO/SILO/LISO/LILO with specific lengths), layer counts (2-64), thread counts, and hardware configurations (Table 1)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper runs inference benchmarks directly via llama-bench."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.3 documents the data pipeline: hardware characterization via synthetic benchmarks, FLOPs calculation methodology (Table 2), memory monitoring strategy per platform, inference workflow with parameterized llama-bench runs, and automated post-hoc analysis from synchronized logs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B ('Future Work') provides substantive discussion of limitations: need to expand to MoE architectures, limited hardware diversity, and software stack variance not investigated. While titled 'Future Work', it functions as a dedicated limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B identifies specific threats: (1) FLOPs estimation for MoE with stochastic routing is a formidable challenge not addressed, (2) different inference engines (TensorRT LLM, vLLM, ONNX Runtime) introduce substantial variance, (3) only dense architectures are covered. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B explicitly states what was NOT tested: MoE architectures, broader hardware range, and different software stacks. The paper also explicitly scopes to 'dense architectures' in the discussion."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw benchmark logs and measurement data are not provided for download. Only aggregated results appear in tables and figures. The GitHub repo appears to contain benchmarking code but not raw data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.3 describes data collection in detail: hardware characterization via PyTorch matrix operations, inference via llama-bench with specific parameters, memory monitoring through platform-specific strategies (RSS sampling, nvidia-smi XML parsing), and timestamp-synchronized logging."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The 'subjects' are well-defined hardware platforms and publicly available model architectures — no recruitment process applies."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.3 documents the full pipeline: hardware profiling → parameterized inference runs → concurrent memory monitoring → JSON-formatted inference logs → automated post-hoc analysis. The FLOPs calculation formulas are provided in Table 2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. There is no mention of funding sources or grants."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Huzhou University, Banbu AI Foundation, Chinese Academy of Sciences, Carnegie Mellon University, and University of Edinburgh."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors affiliated with Banbu AI Foundation evaluate PLM (Deng et al., 2025), which is Banbu AI's own model. PLM (using MLA) is consistently shown as the best-performing architecture. The funder has a direct interest in PLM performing well."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosures statement is present. The paper does not declare whether authors hold patents, equity, or financial interests related to PLM or Banbu AI Foundation."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper measures inference performance (throughput, FLOPS, OI), not model capability or accuracy on benchmarks. Training data contamination is irrelevant to hardware performance profiling."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model capability is being evaluated — the paper profiles inference performance characteristics. Train/test overlap has no bearing on throughput measurements."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark knowledge contamination risk exists. The paper measures hardware utilization and inference speed, not whether models can answer benchmark questions correctly."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a systems performance benchmarking paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. No ethics review is needed for hardware benchmarking."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core contribution IS reporting inference performance: throughput (TPS) in Tables 3-5, GFLOPS across all experiments, latency characteristics across scenarios. Inference cost/performance is the primary output."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware used is specified (Table 1), but the total computational budget for running all experiments (GPU hours, wall-clock time for the full benchmark suite) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or run-to-run variability. Inference benchmarks can exhibit variance from system load, thermal throttling, and memory state, but this is not addressed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per configuration is not stated. It is unclear whether results are from single runs or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter search was performed. The paper systematically evaluates fixed configurations (models, precisions, scenarios, layer counts) without optimization."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not select a 'best' configuration — it characterizes all configurations systematically across the design space."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors from Banbu AI Foundation evaluate PLM (their own model) as the MLA representative, and it consistently outperforms alternatives. The paper does not acknowledge the bias of evaluating their own system or discuss this potential conflict."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly controls for model scale by comparing attention mechanisms at matched ~1.5B parameters (Sec 4.3.2: 'all LLMs are scaled to a unified size of approximately 1.5B parameters by adjusting their respective layer counts'). Performance is shown across matched compute conditions."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper thoroughly discusses what its metrics measure: OI is analytically defined (Eq. 3-5), the Roofline model is a well-established performance analysis framework (Williams et al., 2009), and the relationship between measured metrics and hardware limits is formally established."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The paper benchmarks raw inference performance via llama-bench."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This paper measures inference performance (throughput, FLOPS), not model knowledge or accuracy. Temporal leakage concepts do not apply to hardware performance profiling."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No prediction task is involved. The paper profiles hardware utilization during inference, where feature leakage is structurally irrelevant."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No train/test data split exists in this performance benchmarking study. Independence of data splits is not applicable."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "No leakage risk exists in performance profiling. Detection methods are not applicable."
       }
     }
   },
   "claims": [
     {
       "claim": "Context length is the primary factor determining both the operational intensity and performance of on-device decoding, with LISO approaching compute-bound and SILO remaining severely memory-bound.",
       "evidence": "Figure 2 shows LISO (yellow dots) consistently achieving highest OI and proximity to the Roofline ridge point across all models, while SILO (red dots) remains deep in the memory-bound regime. Demonstrated across Qwen2.5, Llama 3.2, PLM, and SmolLM2 on Apple M1 Pro (Sec 4.1).",
       "supported": "strong"
     },
     {
       "claim": "On-device decoding reaches peak operational intensity at a remarkably shallow depth (3-5 layers), beyond which OI regresses due to memory bandwidth overhead.",
       "evidence": "Figure 3 shows a non-monotonic 'arch' trajectory of OI as layer count increases from 2 to 64 across FP16, Q8_0, and Q4_K_M precisions on Apple M1 Pro. The inflection occurs at 3-5 layers (Sec 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Quantization provides maximal efficiency gains for memory-bound tasks but yields diminishing returns in compute-heavy scenarios.",
       "evidence": "Figure 4 shows FP16→Q4_K_M yields dramatic OI and GFLOPS gains in SILO (memory-bound) but tight clustering in LISO (near compute-bound). Demonstrated across all 6 model architectures (Sec 4.3.1).",
       "supported": "strong"
     },
     {
       "claim": "MLA outperforms both MHA and GQA by effectively compressing KV cache traffic to maximize operational intensity on resource-constrained devices.",
       "evidence": "Figure 6 shows MLA (PLM, blue dots) consistently achieving highest OI and GFLOPS across all four sequence scenarios at matched ~1.5B parameters on Apple M1 Pro. Cross-platform comparison in Figure 5 (right) confirms on RTX 3070Ti (Sec 4.3.2, 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Hardware heterogeneity creates an 'efficiency trap' where a single model architecture cannot achieve uniform utilization equity across platforms.",
       "evidence": "Figure 5 (left) shows ridge points spanning 8.98 (Raspberry Pi 5) to 38.00 (RTX 3090) FLOPs/Byte, with peak performance spanning three orders of magnitude. The same 1.5B model operates in different bottleneck regimes on different hardware (Sec 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Architectural optimization (MLA) demonstrates consistent cross-platform robustness, maintaining a higher baseline of operational intensity across hardware tiers.",
       "evidence": "Figure 5 (right) shows MLA maintaining superior position on both Apple M1 Pro and RTX 3070Ti Laptop in LISO scenario, with a parallel shift toward upper-right rather than convergence (Sec 5.2).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RooflineBench characterizes LLM inference on edge devices using the Roofline model, revealing that context length is the primary determinant of operational intensity and that OI peaks at 3-5 layers before regressing. The framework identifies an 'efficiency trap' from hardware ridge-point disparities across heterogeneous platforms. Multi-head Latent Attention (MLA) consistently achieves the highest operational intensity and cross-platform robustness compared to MHA and GQA at matched model scales, demonstrating how architectural refinements can bridge the gap between theoretical potential and real-world execution.",
   "red_flags": [
     {
       "flag": "Self-evaluation conflict of interest",
       "detail": "Authors affiliated with Banbu AI Foundation evaluate PLM (Deng et al., 2025), which is Banbu AI's own model. PLM (MLA) is consistently shown as the best-performing architecture throughout the paper. This conflict is never disclosed or acknowledged."
     },
     {
       "flag": "No error bars or run-to-run variance",
       "detail": "All results are single point estimates without any uncertainty quantification. Inference benchmarks can vary significantly due to system load, thermal throttling, memory state, and OS scheduling. The paper makes strong comparative claims ('MLA consistently achieves the highest') without statistical validation."
     },
     {
       "flag": "Potentially unfair MLA comparison",
       "detail": "The MLA representative is PLM-1.8B while GQA uses Qwen2.5-1.5B. Although matched to ~1.5B by adjusting layer counts, the architectures differ in hidden dimension, head count, and other design choices beyond just the attention mechanism. Attributing the performance difference solely to the attention type oversimplifies the comparison."
     }
   ],
   "cited_papers": [
     {
       "title": "PaLM: Scaling language modeling with pathways",
       "authors": ["Aravind Chowdhery"],
       "year": 2023,
       "relevance": "Introduces Model FLOPs Utilization (MFU) metric used to evaluate LLM training and inference efficiency."
     },
     {
       "title": "Efficiently scaling transformer inference",
       "authors": ["Reiner Pope"],
       "year": 2023,
       "relevance": "Analytical framework for Transformer inference that RooflineBench builds upon for decomposing prefill and decoding phases."
     },
     {
       "title": "FlashAttention: Fast and memory-efficient exact attention with IO-awareness",
       "authors": ["Tri Dao"],
       "year": 2022,
       "arxiv_id": "2205.14135",
       "relevance": "IO-aware attention optimization for memory-efficient inference, directly relevant to on-device LLM efficiency."
     },
     {
       "title": "DeepSeek-V2: A strong, economical, and efficient mixture-of-experts language model",
       "authors": ["DeepSeek-AI"],
       "year": 2024,
       "arxiv_id": "2405.04434",
       "relevance": "Introduces Multi-head Latent Attention (MLA) evaluated in RooflineBench as the most efficient attention architecture."
     },
     {
       "title": "MiniCPM: Unveiling the potential of small language models with scalable training strategies",
       "authors": ["Shengding Hu"],
       "year": 2024,
       "arxiv_id": "2404.06395",
       "relevance": "Small language model with trainable sparse attention, demonstrating efficient on-device inference capabilities."
     },
     {
       "title": "Phi-3 technical report: A highly capable language model locally on your phone",
       "authors": ["Marah I. Abdin"],
       "year": 2024,
       "arxiv_id": "2404.14219",
       "relevance": "Demonstrates compact LLM architecture capable of on-device deployment, directly relevant to on-device intelligence evaluation."
     },
     {
       "title": "AWQ: activation-aware weight quantization for on-device LLM compression and acceleration",
       "authors": ["Ji Lin"],
       "year": 2024,
       "relevance": "Post-training quantization technique for on-device LLM inference, relevant to efficiency-focused evaluation."
     },
     {
       "title": "GPTQ: accurate post-training quantization for generative pre-trained transformers",
       "authors": ["Elias Frantar"],
       "year": 2022,
       "arxiv_id": "2210.17323",
       "relevance": "Post-training quantization method for reducing LLM memory footprint, directly relevant to on-device deployment."
     },
     {
       "title": "LLM in a flash: Efficient large language model inference with limited memory",
       "authors": ["Keivan Alizadeh"],
       "year": 2024,
       "doi": "10.18653/V1/2024.ACL-LONG.678",
       "relevance": "Addresses memory-constrained LLM inference showing the 'memory wall' bottleneck for generative models on edge devices."
     },
     {
       "title": "MobileLLM: Optimizing sub-billion parameter language models for on-device use cases",
       "authors": ["Zechun Liu"],
       "year": 2024,
       "relevance": "Compact architecture optimized for mobile deployment, representative of the SLM trend evaluated in this paper."
     },
     {
       "title": "Holistic evaluation of language models",
       "authors": ["Percy Liang"],
       "year": 2023,
       "relevance": "Comprehensive multi-metric LLM evaluation framework (HELM), relevant as a capability-centric benchmark that contrasts with system-level evaluation."
     },
     {
       "title": "MLPerf inference benchmark",
       "authors": ["Vijay Janapa Reddi"],
       "year": 2019,
       "relevance": "Industry-standard inference benchmarking protocol for heterogeneous hardware, foundational for system-level AI evaluation."
     }
   ]
 }