ai-research-survey

scan.json (25770B)

---

 {
   "paper": {
     "title": "OJBKQ: Objective-Joint Babai-Klein Quantization",
     "authors": [
       "Xinyu Wang",
       "Ziyu Zhao",
       "Peng Lu",
       "Yu Gu",
       "Xiao-Wen Chang"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.08376"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OJBKQ reformulates layer-wise post-training quantization as a box-constrained integer least squares problem and solves it using an extended Klein randomized Babai algorithm with K parallel candidate paths. Combined with the Joint Target Alignment (JTA) scoring objective that interpolates between runtime-quantized and full-precision references, OJBKQ consistently achieves lower perplexity and higher accuracy than GPTQ, AWQ, and QUIP at 3-4 bits across LLaMA, Qwen3, and Mistral models. The advantage is more pronounced at 3-bit settings where prior methods often fail catastrophically.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, GitHub link, or archive URL is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation datasets are publicly available standard benchmarks: C4, WikiText-2, ARC, BoolQ, HellaSwag, PIQA, WinoGrande, GSM8K, GPQA, MBPP. All models used are publicly released (LLaMA, Qwen3, Mistral)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment setup section is provided. Hardware and software dependencies are not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts to replicate experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3 are point estimates with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims OJBKQ 'outperforms' and 'achieves the lowest' perplexity based on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-3 present absolute performance numbers for both baselines and the proposed method side by side (e.g., GPTQ 9.41 vs Ours 9.33 on C4), providing sufficient context for the reader to assess the magnitude of improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The calibration set is 128 samples from C4 with sequence length 2048, but no justification is given for why 128 samples is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are single-run numbers with no standard deviation, variance, or spread measures reported. This is particularly concerning given the method uses randomized rounding (Klein's algorithm)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against RTN, GPTQ, AWQ, and QUIP as baselines (Section 4, Tables 1-3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPTQ (2023), AWQ (2024), and QUIP (2024) are contemporary state-of-the-art PTQ methods. The paper also references their most recent versions."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates components incrementally: Ours(N) = Naive Babai only, Ours(R) = Random-K Babai, Ours = Random-K with JTA. Additional ablations on K (Figure 2) and µ/λ hyperparameters (Figure 3, Table 4) are provided."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses perplexity (C4, WikiText-2), zero-shot accuracy (ARC, BoolQ, HellaSwag, PIQA, WinoGrande), and reasoning accuracy (GSM8K, GPQA, MBPP)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for evaluating quantization method quality, which is measured by model performance on standard benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Calibration uses 128 samples from C4, while evaluation is performed on standard test splits of C4, WikiText-2, and other benchmarks. Zero-shot and reasoning tasks use the lm-harness library with standard evaluation protocols."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model family (LLaMA-2, LLaMA-3, Qwen3, Mistral), model size (0.6B to 13B), bit-width (3-bit, 4-bit), group size (g128, no grouping), and individual benchmark tasks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss cases where the method underperforms. For example, Ours(N) gives 47.23/42.75 on Q3-0.6B at 3-bit (worse than GPTQ's 43.60/41.54), but this is not discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every presentation frames the method positively. Cases where the method is worse than baselines (e.g., Ours(N) on Qwen3-0.6B/4B at 3-bit) are present in tables but not acknowledged or discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'lower perplexity at 3–4 bits compared to existing PTQ approaches' — supported by Table 1 across most settings. 'Comparable computational cost' — supported by Figure 4 showing modest overhead at K=5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('Random-K expands the quality of the candidate set, while JTA improves how we select') are justified through the incremental ablation: Ours(N) → Ours(R) → Ours, which isolates each component's contribution via controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract bounds claims to 'large language models' at '3–4 bits,' and the experiments test specific model families (LLaMA-2/3, Qwen3, Mistral) at those bit widths. The intro further scopes to 'standard and practically relevant 4-bit and 3-bit settings (e.g., group size 128).'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed improvements are discussed. The paper does not consider whether improvements could be due to specific model architectures, calibration data selection, or other factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures perplexity and accuracy on standard benchmarks and claims improvements in quantization quality — no proxy gap exists. Claims match the granularity of measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names and sizes are provided: LLaMA-2-7B, LLaMA-2-13B, LLaMA-3-8B, Qwen3-0.6B, Qwen3-4B, Qwen3-8B, Mistral-7B. For open-weight models, the model name and size uniquely identify the checkpoint."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a weight quantization method that operates on model parameters, not through prompting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: K=5, µ=0.6/λ=0.6 for 3-bit, µ=0.1/λ=0.2 for 4-bit, group size 128, calibration: 128 samples from C4 with seq length 2048, α formula given in Eq. 13."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a numerical optimization method for weight quantization."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Calibration procedure is described: '128 samples from the C4 dataset with a sequence length of 2048 tokens.' Scale and zero-point computation uses 'standard statistical calibration methods (e.g., the Absmax method).' Evaluation uses lm-harness library."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' paragraph appears at the end of Section 5 (Conclusion), discussing specific gaps in the current framework."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section lists method extensions (weight permutation, dynamic scaling, per-layer adaptive hyperparameters) rather than threats to the validity of the experimental conclusions. No discussion of whether results could be sensitive to calibration data choice, random seed, or other experimental factors."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations describe what the framework 'does not yet incorporate' (future work) rather than explicitly stating what the results do NOT show or what claims the authors are NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data, logs, or intermediate results are made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are clearly described: publicly available pre-trained models, C4 calibration samples, and standard evaluation benchmarks with references provided for each."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and pre-trained models."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: calibration data selection (128 C4 samples, 2048 tokens), quantization procedure (Absmax for scales/zeros, layer-wise JTA optimization), and evaluation protocol (perplexity computation, lm-harness for zero-shot/reasoning tasks)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: McGill University and Université de Montréal. These are academic institutions not affiliated with the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed. Absence of disclosure is not evidence of independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models (LLaMA-2/3, Qwen3, Mistral). These models were likely trained on data that includes many of the evaluation benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained models may have seen C4, WikiText-2, ARC, HellaSwag, or other evaluation benchmarks during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "WikiText-2, ARC, HellaSwag, PIQA, WinoGrande, and other benchmarks predate all evaluated models' training periods. No contamination risk is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Figure 4 shows relative per-layer quantization time increase ratios for different K values, but no absolute wall-clock times, GPU hours, or inference latency numbers are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, hardware specifications, or total runtime for the experiments is stated anywhere in the paper."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run. No seed sensitivity analysis is reported, despite the method using randomized rounding (Klein's algorithm) which should exhibit seed dependence."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per result is never stated. Results are presented as single values without indicating whether they were averaged over multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 shows a grid search over µ and λ (10×8 = 80 configurations) and Figure 2 shows K ablation, but the total compute spent on hyperparameter search is not stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper justifies selecting (µ=0.6, λ=0.6) over the global minimum (µ=0.6, λ=0.4): 'the configuration (µ=0.6, λ=0.6) achieves comparable performance (PPL 7.87) and lies within a stable low-perplexity region.' The full grid is shown in Table 4."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so there are no p-values to correct for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own method and compare against baselines using 'default configurations of each baseline's repo,' but do not acknowledge the bias of evaluating their own system or discuss whether their baseline implementations are competitive."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows perplexity vs K (number of candidates), and Figure 4 shows per-layer time increase ratio vs K. Together these show the performance-compute tradeoff for the key hyperparameter."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether perplexity on C4/WikiText-2 or zero-shot accuracy on standard benchmarks actually measure the quality of quantization that matters for real-world deployment."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. This is a weight quantization method evaluated on standard benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained models' training data includes information from the benchmark datasets, many of which predate the models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The paper does not address whether calibration data selection could influence results on related evaluation sets."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between calibration data (C4 samples) and evaluation data (C4 test set). Both are drawn from the same corpus."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used or discussed."
       }
     }
   },
   "claims": [
     {
       "claim": "OJBKQ achieves lower perplexity at 3-4 bits compared to existing PTQ approaches (GPTQ, AWQ, QUIP) across multiple LLM families.",
       "evidence": "Table 1 shows OJBKQ achieves the lowest perplexity in most settings across LLaMA-2/3, Qwen3, and Mistral models. For example, on LLaMA-3-8B at 4-bit g128: GPTQ 9.41/6.54, AWQ 9.40/6.54, Ours 9.33/6.48 (C4/WikiText-2).",
       "supported": "strong"
     },
     {
       "claim": "The computational overhead of Random-K decoding is modest, with approximately 80% increase in per-layer runtime at K=25.",
       "evidence": "Figure 4 shows relative time increase ratios per layer for K=1,5,10,25. At K=5 (default setting), overhead is approximately 15-20%.",
       "supported": "moderate"
     },
     {
       "claim": "Random-K exploration improves over deterministic Babai rounding by finding superior integer candidates.",
       "evidence": "Tables 1-3 consistently show Ours(R) improving over Ours(N). Figure 1 shows JTA reconstruction error decreasing with higher K across layers and modules.",
       "supported": "strong"
     },
     {
       "claim": "The JTA objective provides a better candidate selection criterion than using either runtime-quantized or full-precision references alone.",
       "evidence": "Tables 1-3 show Ours (with JTA) consistently outperforming Ours(R) (without JTA). Figure 3 shows a U-shaped curve for µ, confirming neither extreme (µ=0 or µ=1) is optimal. The ablation in Table 4 further demonstrates this across a grid of µ and λ values.",
       "supported": "strong"
     },
     {
       "claim": "The advantage of OJBKQ becomes more pronounced at 3-bit quantization compared to 4-bit.",
       "evidence": "Table 1 shows larger performance gaps at 3-bit: on LLaMA-3-8B without grouping, GPTQ degrades to 29.42/23.81 while Ours achieves 11.38/8.04 on C4/WikiText-2. At 4-bit the gaps are much smaller.",
       "supported": "strong"
     },
     {
       "claim": "OJBKQ preserves reasoning capabilities under 4-bit quantization, closely tracking BF16 accuracy.",
       "evidence": "Table 3 shows Ours achieving 84.29% on GSM8K for Qwen3-4B (vs 83.70% BF16) and 48.22% on GSM8K for LLaMA3-8B (vs 51.86% BF16). Average reasoning accuracy is competitive with or better than all baselines.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars despite randomized method",
       "detail": "The method uses Klein's randomized algorithm with K=5 trials, inherently producing variable results. Yet all experiments report single-run point estimates with no standard deviation, variance, or confidence intervals. The reader cannot assess whether observed improvements are within noise."
     },
     {
       "flag": "Selective presentation of results",
       "detail": "Ours(N) performs worse than GPTQ on Qwen3-0.6B at 3-bit (47.23 vs 43.60 on C4) and Ours(N) gives 25.46 vs GPTQ's 18.16 on Q3-4B at 3-bit g128 W3A16, but these regressions are not discussed. The paper only highlights cases where the method wins."
     },
     {
       "flag": "No code release",
       "detail": "The method involves non-trivial GPU kernel design (PPI-KBabai) and specific implementation choices, but no code is released, making independent reproduction difficult."
     },
     {
       "flag": "Calibration-evaluation overlap",
       "detail": "C4 is used for both calibration (128 samples) and perplexity evaluation. While the calibration set is small relative to the test set, this overlap is not discussed or analyzed for potential bias."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM whose quantization is a key deployment concern; one of the models motivating compression research."
     },
     {
       "title": "GPTQ: Accurate post-training quantization for generative pre-trained transformers",
       "authors": ["Elias Frantar", "Saleh Ashkboos", "Torsten Hoefler", "Dan Alistarh"],
       "year": 2023,
       "relevance": "Primary PTQ baseline; established the sequential rounding approach for LLM quantization that this paper improves upon."
     },
     {
       "title": "AWQ: Activation-aware weight quantization for LLM compression and acceleration",
       "authors": ["Ji Lin", "Jiaming Tang", "Haotian Tang"],
       "year": 2024,
       "relevance": "Key PTQ baseline using salience-based scaling for LLM compression."
     },
     {
       "title": "QUIP: 2-bit quantization of large language models with guarantees",
       "authors": ["Jerry Chee", "Yaohui Cai", "Volodymyr Kuleshov", "Christopher De Sa"],
       "year": 2024,
       "relevance": "Rotation-based PTQ method providing theoretical quantization guarantees for LLMs."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Addresses LLM deployment cost reduction, the same practical concern motivating quantization research."
     },
     {
       "title": "The geometry of LLM quantization: GPTQ as Babai's nearest plane algorithm",
       "authors": ["Jiale Chen", "Yalda Shabanzadeh", "Elvir Crnčević", "Torsten Hoefler", "Dan Alistarh"],
       "year": 2025,
       "relevance": "Establishes the theoretical connection between PTQ and lattice decoding that this paper builds upon."
     },
     {
       "title": "Quantization error propagation: Revisiting layer-wise post-training quantization",
       "authors": ["Yamato Arai", "Yuma Ichikawa"],
       "year": 2025,
       "relevance": "Directly motivates the JTA objective by analyzing how layer-wise quantization errors propagate."
     },
     {
       "title": "Benchmarking post-training quantization in LLMs: Comprehensive taxonomy, unified evaluation, and comparative analysis",
       "authors": ["Jiaqi Zhao", "Ming Wang", "Miao Zhang"],
       "year": 2025,
       "relevance": "Comprehensive PTQ survey that provides the taxonomy used to classify quantization methods in this paper."
     },
     {
       "title": "LLaMA: Open and efficient foundation language models",
       "authors": ["Hugo Touvron"],
       "year": 2023,
       "arxiv_id": "2302.13971",
       "relevance": "Key open-weight LLM family used as primary evaluation target for quantization methods."
     },
     {
       "title": "Mistral 7B",
       "authors": ["Albert Q. Jiang"],
       "year": 2023,
       "relevance": "Open-weight LLM used as evaluation target, representing diverse model architectures for quantization."
     }
   ]
 }