ai-research-survey

scan.json (24839B)

---

 {
   "paper": {
     "title": "LUT-LLM: Efficient Large Language Model Inference with Memory-based Computations on FPGAs",
     "authors": [
       "Zifan He",
       "Shengyu Ye",
       "Rui Ma",
       "Yang Wang",
       "Jason Cong"
     ],
     "year": 2025,
     "venue": "arXiv preprint (submitted to ACM conference)",
     "arxiv_id": "2511.06174"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The paper mentions using TAPA framework and Vivado tools but does not release its own source code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: GLUE benchmark, SQuAD v2, MMLU-Pro, and the FineWeb dataset for training. The Qwen 3 model is also publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 specifies the FPGA (AMD Alveo V80), Xilinx Vitis HLS 2024.2, TAPA framework, RapidStream for floorplanning, Vivado 2024.2, and GPU benchmarking setup with vLLM. Hardware configurations are detailed in Table 2."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the design methodology and tools used but does not include commands or scripts to reproduce the results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results. Latency and energy efficiency numbers are presented as point estimates without uncertainty bounds."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., '1.66x lower latency than MI210') without any statistical significance tests. Comparisons are based solely on point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as speedup ratios with baseline context throughout (e.g., '1.66x lower geomean latency than MI210', '1.72x more energy efficient than A100', '5.6x and 1.9x faster than Allo and InTAR'). These ratios provide clear magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the range of input/output configurations tested. The context window is limited to 512 tokens due to training constraints but no power analysis or sample size justification is given for the benchmark evaluation points."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across experimental runs. All latency and energy efficiency results appear to be single-run measurements without reporting any spread."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: GPUs (AMD MI210, NVIDIA A100) with both BF16 and INT8 precision, and FPGA baselines (Allo, InTAR, FlightLLM). Quantization baselines (SmoothQuant, GPTQ, RTN INT8) are also compared for model quality."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent works: Allo (PLDI 2024), InTAR (FCCM 2025), FlightLLM (FPGA 2024), FlashAttention-2 (2023), FlashDecoding (2023), GPTQ Marlin kernel (2025). These represent the current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates the quantization scheme: Table 3 shows progressive quality impact of activation quantization, INT8 LUT quantization, and weight quantization. Section 3 compares weight-only, activation-only, and activation-weight co-quantization performance models. Section 4 compares pure dataflow vs. sequential vs. hybrid execution strategies."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: end-to-end latency, decode latency, energy efficiency (tokens/Joule), and model quality metrics (accuracy and F1 across GLUE sub-tasks, SQuAD v2, MMLU-Pro). Resource utilization is also reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for this hardware accelerator paper. The evaluation concerns latency, energy efficiency, and model quality on standard benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Model quality is evaluated on standard benchmarks (GLUE, SQuAD v2, MMLU-Pro) which serve as held-out test sets. The model is trained on FineWeb and finetuned on WikiQA, then tested on these separate benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-task breakdown across 7 GLUE sub-tasks and SQuAD v2. Figures 11 and 12 break down performance by input/output length combinations. Latency is broken down into prefill and decode stages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where LUT-LLM underperforms: it is slower than A100 with GPTQ Marlin kernel (Section 5.3.2), the 512-token context window limitation due to training memory constraints is acknowledged (Section 5.1), and prefill performance narrows against InTAR as input length increases (Section 5.3.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 reports that naive activation-only VQ is 'highly memory bounded for decoding' and 'unfavorable for tasks involving short inputs or long outputs.' The paper explicitly identifies that activation-only quantization loads 16x larger data than FP16 weights, motivating the co-quantization approach."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 1.66x lower end-to-end latency than MI210 and 1.72x higher energy efficiency than A100 are supported by Figures 11 and 12 and the detailed results in Section 5.3.2. The 2.16x energy efficiency for 32B model scaling is discussed in Section 5.3.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about why co-quantization outperforms other schemes are supported by the performance model in Section 3 with detailed mathematical derivations. The ablation of quantization schemes and execution strategies in Sections 3.2 and 4.4 provides controlled single-variable analysis."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Efficient Large Language Model Inference' implies broad applicability, but results are demonstrated only on a customized Qwen 3 1.7B model with a 512-token context window on a single FPGA. The 32B scaling result in Section 5.3.4 is only a performance model projection, not a measured result. The paper does not explicitly bound its claims to the tested configuration."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed performance gains. For example, it does not consider whether the advantages come primarily from the process node differences, different power envelopes, or specific properties of the Qwen 3 architecture rather than the LUT-LLM approach itself."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Qwen 3 1.7B' as the target model and provides exact hardware part numbers: AMD Alveo V80 FPGA, AMD Instinct MI210, NVIDIA A100. Tool versions are specified: Xilinx Vitis HLS 2024.2, Vivado 2024.2."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting as part of its methodology. It is a hardware accelerator paper that evaluates inference latency and model quality on standard benchmarks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports quantization hyperparameters (G=512, v=2, cw=16, ca=64, INT8 lookup tables, equivalent to W2A3), training setup (reconstruction loss ratio 0.1, STE with adjustable gradients, 3 epochs on WikiQA, 512-token sequences), and the achieved clock frequency (227 MHz)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a hardware accelerator paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 documents the quantization-aware training pipeline: pretrain on FineWeb, finetune on WikiQA, apply activation vector quantization with two-stage training strategy from LUT-DLA, reconstruct weights, apply GPTVQ for weight quantization, then pre-compute 2D lookup tables. Per-tensor zero-point quantization is described with equations."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Some limitations are mentioned in passing (512-token context window in Section 5.1, scaling discussion in Section 5.3.4) but there is no structured limitations or threats-to-validity section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Potential concerns such as FPGA vs GPU comparison fairness (different process nodes, power budgets, software maturity), simulator accuracy for FPGA baselines, and the customized model not being a standard Qwen 3 checkpoint are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show, such as that the 32B scaling is projected rather than measured, or that results may not generalize to other model architectures or longer context lengths."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (latency traces, power measurements, detailed per-layer timing) is available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes GPU benchmarking methodology: vLLM backend, FlashAttention and FlashDecoding enabled, GPTQ INT8 quantization, power monitoring via pyNVML and pyrsmi. FPGA baseline simulation methodology is described with 'less than 2% gap from data in the original papers.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates hardware on standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline from model training through quantization to hardware deployment is documented in Section 5.1: pretrain on FineWeb -> finetune on WikiQA -> activation VQ training -> weight reconstruction -> GPTVQ -> 2D lookup table computation -> FPGA deployment. GPU benchmarking pipeline is also described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section mentioning grants or sponsors is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed: UCLA and Microsoft Research Asia. The footnote notes 'Work done during internship at Microsoft.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three of five authors are from Microsoft Research Asia, which has commercial interest in efficient LLM inference. The AMD V80 FPGA is the target platform, and AMD hardware is used for comparison. No discussion of whether these affiliations create conflicts."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen 3 1.7B but does not state its training data cutoff date. This matters because the model is evaluated on GLUE, SQuAD v2, and MMLU-Pro."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. GLUE and SQuAD v2 are old benchmarks (2018 and 2016) that could easily be in the training data of Qwen 3."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GLUE (2018) and SQuAD v2 (2016) were available well before Qwen 3's training. MMLU-Pro (2024) is more recent but still likely pre-dates training. No contamination discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference latency is reported in milliseconds across multiple configurations (Figures 11, 14). Energy efficiency is reported in tokens per Joule (Figure 12, Table 5). Power consumption of the FPGA (peak 190W) and GPUs (300W) is stated in Table 2."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states training cost: 'an eight-GPU cluster will take 2 weeks to first pretrain Qwen 3 1.7B on FineWeb for 512-token sequences and then finetune on WikiQA with 3 epochs, with 75% memory utilization.' FPGA resource utilization is detailed in Figure 13."
       }
     }
   },
   "claims": [
     {
       "claim": "LUT-LLM achieves 1.66x lower end-to-end latency than the AMD MI210 GPU for Qwen 3 1.7B inference.",
       "evidence": "Figure 11 shows latency comparison across input/output length configurations. Section 5.3.2 reports 'LUT-LLM has a 1.66x lower geomean latency than MI210, despite utilizing 7x less memory bandwidth.'",
       "supported": "strong"
     },
     {
       "claim": "LUT-LLM is 1.72x more energy efficient than the NVIDIA A100 GPU.",
       "evidence": "Figure 12 shows energy efficiency comparisons. Section 5.3.2 states 'LUT-LLM remains 1.72x more energy efficient in geomean.'",
       "supported": "strong"
     },
     {
       "claim": "LUT-LLM is 4.1x more energy efficient than AMD MI210.",
       "evidence": "Section 5.3.2 and Figure 12 report '4.1x higher geomean energy efficiency than MI210.'",
       "supported": "strong"
     },
     {
       "claim": "Activation-weight co-quantization achieves the best performance across both prefill and decode stages.",
       "evidence": "Figure 6 compares normalized throughputs of various quantization schemes. Section 3.2 provides performance model analysis showing co-quantization has superior throughput. Section 3.1 derives that co-quantization gives 569 cycles vs. 8256 for activation-only and 1090 for weight-only in the worked example.",
       "supported": "strong"
     },
     {
       "claim": "LUT-LLM's quantization incurs only 2.7% performance drop relative to FP16 baseline.",
       "evidence": "Table 3 shows model quality across GLUE sub-tasks and SQuAD v2 for different quantization schemes. Section 5.2 states '2.7% performance drop relative to the FP16 baseline with all techniques applied.'",
       "supported": "moderate"
     },
     {
       "claim": "LUT-LLM can scale to a 32B model with 2.16x better energy efficiency than the A100.",
       "evidence": "Section 5.3.4 mentions this claim but it is based on performance model projection, not actual implementation. The paper states 'the current training algorithm limits the model size' and describes extending 'with simple changes of loop bounds.'",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "LUT-LLM is the first FPGA accelerator for 1B+ language model inference using memory-based computation via vector quantization. The paper demonstrates that activation-weight co-quantization with 2D lookup tables outperforms weight-only or activation-only quantization by reducing memory bandwidth demands and on-chip port requirements. On AMD V80 FPGA running a customized Qwen 3 1.7B model, LUT-LLM achieves 1.66x lower latency than AMD MI210 and 1.72x higher energy efficiency than NVIDIA A100, while maintaining competitive model quality (2.7% drop from FP16 baseline).",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All performance results (latency, energy efficiency) are reported as single point estimates without any error bars, confidence intervals, or variance from multiple runs. Hardware measurements can vary due to thermal throttling, power states, and other factors."
     },
     {
       "flag": "Unfair FPGA baseline comparison",
       "detail": "The FPGA baselines (Allo, InTAR, FlightLLM) are from the older AMD U280 FPGA, while LUT-LLM uses the newer AMD V80. The paper constructs a simulator for these baselines 'with less than 2% gap from data in the original papers' rather than running them on the same hardware, introducing potential simulation bias."
     },
     {
       "flag": "Limited context window",
       "detail": "Results are limited to 512-token context windows due to training memory constraints. This is a significant practical limitation not prominently highlighted, as modern LLMs routinely handle 4K-128K tokens."
     },
     {
       "flag": "32B scaling claim is projection only",
       "detail": "The claim of 2.16x energy efficiency over A100 for 32B models (in the abstract) is based on a performance model projection, not actual implementation and measurement. This is not clearly distinguished from measured results."
     },
     {
       "flag": "No contamination analysis for model quality benchmarks",
       "detail": "Model quality is evaluated on GLUE (2018) and SQuAD v2 (2016) which are likely in Qwen 3's training data. Without contamination analysis, the quality preservation claims may be overstated."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated discussion of limitations or threats to validity, despite several notable limitations (single model, single FPGA, limited context, simulated baselines, customized model)."
     }
   ],
   "cited_papers": [
     {
       "title": "FlightLLM: Efficient Large Language Model Inference with a Complete Mapping Flow on FPGAs",
       "authors": ["Shulin Zeng", "Jun Liu", "Guohao Dai"],
       "year": 2024,
       "relevance": "State-of-the-art FPGA LLM accelerator serving as a key baseline for hardware performance comparison."
     },
     {
       "title": "Allo: A Programming Model for Composable Accelerator Design",
       "authors": ["Hongzheng Chen", "Niansong Zhang", "Shaojie Xiang"],
       "year": 2024,
       "relevance": "FPGA accelerator framework for LLMs serving as a baseline, demonstrates composable accelerator design methodology."
     },
     {
       "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention",
       "authors": ["Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang"],
       "year": 2023,
       "relevance": "Introduces vLLM framework used as the GPU benchmarking backend in this work."
     },
     {
       "title": "GPTQ: Accurate Post-Training Quantization for Generative Pre-Trained Transformers",
       "authors": ["Elias Frantar", "Saleh Ashkboos", "Torsten Hoefler", "Dan Alistarh"],
       "year": 2022,
       "arxiv_id": "2210.17323",
       "relevance": "Key quantization method used for GPU baselines and as a comparison approach for model quality."
     },
     {
       "title": "SmoothQuant: Accurate and Efficient Post-Training Quantization for Large Language Models",
       "authors": ["Guangxuan Xiao", "Ji Lin", "Mickael Seznec"],
       "year": 2023,
       "relevance": "Scalar quantization baseline compared for model quality preservation under aggressive quantization."
     },
     {
       "title": "LUT-NN: Empower Efficient Neural Network Inference with Centroid Learning and Table Lookup",
       "authors": ["Xiaohu Tang", "Yang Wang", "Ting Cao"],
       "year": 2023,
       "relevance": "Foundational work on lookup-table-based neural network inference using vector quantization and centroid learning."
     },
     {
       "title": "LUT-DLA: Lookup Table as Efficient Extreme Low-Bit Deep Learning Accelerator",
       "authors": ["Guoyu Li", "Shengyu Ye", "Chunyun Chen"],
       "year": 2025,
       "relevance": "ASIC-based LUT accelerator design that LUT-LLM builds upon, providing the two-stage training strategy for centroids and weights."
     },
     {
       "title": "T-MAC: CPU Renaissance via Table Lookup for Low-Bit LLM Deployment on Edge",
       "authors": ["Jianyu Wei", "Shijie Cao", "Ting Cao"],
       "year": 2025,
       "relevance": "CPU-based table lookup approach for LLM inference, demonstrating memory-based computation on different hardware platforms."
     },
     {
       "title": "GPTVQ: The Blessing of Dimensionality for LLM Quantization",
       "authors": ["Mart Van Baalen", "Andrey Kuzmin", "Ivan Koryakovskiy"],
       "year": 2024,
       "arxiv_id": "2402.15319",
       "relevance": "Vector post-training quantization method for LLMs used in the weight quantization step of LUT-LLM's pipeline."
     },
     {
       "title": "FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning",
       "authors": ["Tri Dao"],
       "year": 2023,
       "arxiv_id": "2307.08691",
       "relevance": "Key GPU optimization enabling competitive GPU baselines for LLM inference performance comparison."
     },
     {
       "title": "Understanding the Potential of FPGA-based Spatial Acceleration for Large Language Model Inference",
       "authors": ["Hongzheng Chen", "Jiahao Zhang", "Yixiao Du"],
       "year": 2024,
       "relevance": "Foundational analysis of FPGA-based LLM acceleration that provides the performance modeling methodology built upon in this work."
     }
   ]
 }