ai-research-survey

scan-v5.json (25561B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "On the Limits of Layer Pruning for Generative Reasoning in LLMs",
     "authors": [
       "Safal Shrestha",
       "Anubhav Shrestha",
       "Aadim Nepal",
       "Minwu Kim",
       "Keith Ross"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.01997",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are verified by Tables 1–3 and Figures 1–8: classification vs. generative gap, SGR achieving up to 90% classification retention and 20–30pp generative gains, and fundamental recovery limits.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims (layer removal degrades arithmetic; SGR improves recovery) and supports them through single-layer ablations, controlled arithmetic probes, and multi-model comparisons across 4 architectures.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to instruct-tuned 7–8B models under constrained post-training regimes; the paper explicitly limits generalization to settings without pretraining-scale data or compute.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly tests and rejects text degeneration as a sufficient explanation for reasoning failures, and Sections 4.2–4.3 identify arithmetic and syntactic degradation as the true root causes.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes benchmark accuracy (GSM8K, HumanEval+) from underlying capabilities (arithmetic logprob accuracy, parenthesis tracking error rates), and uses controlled arithmetic probes to isolate specific abilities.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations section; limitations are woven into the combined 'Discussion & Conclusion' (Section 7), which does not meet the criterion of a dedicated section.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No formal threats-to-validity discussion is present; the paper acknowledges model-family variability and scope restriction to constrained settings, but does not enumerate specific threats such as benchmark contamination or generalization across model scales.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds claims to 'realistic post-training constraints, without access to pretraining-scale data or compute' and notes results are limited to the 4 model families tested.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements state: 'NYU Abu Dhabi Center for Artificial Intelligence and Robotics, funded by Tamkeen under the Research Institute Award CG010.'",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as from the Department of Computer Science, New York University Abu Dhabi; no affiliation with LLM vendors being evaluated.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Tamkeen is a UAE government research funding body with no commercial stake in LLM compression outcomes.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "The Impact Statement contains no competing interests declaration; no statement about patents, equity, or consulting relationships is present anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are operationally defined: classification benchmarks are log-likelihood scoring tasks; generative benchmarks require multi-token solution generation; BI and Reverse Order pruning strategies are explained in Appendix A.6.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 lists four explicit bullet-point contributions: sensitivity characterization, systematic failure mode analysis, SGR recovery strategy, and post-recovery arithmetic/syntactic analysis.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 explicitly positions this work against prior approaches (knowledge distillation, continued pretraining, lightweight module replacement), explaining why existing methods are insufficient and what gap this paper fills.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Code is available at https://github.com/safal312/on-the-limits-of-layer-pruning, linked in the paper header footnote.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All evaluation benchmarks (GSM8K, HumanEval+, MBPP+, XSUM, HellaSwag, PIQA, etc.) are standard publicly available datasets; models and data also noted as released on HuggingFace.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Appendix A.6 mentions single A100 80GB GPU and QLoRA with NF4 quantization, but no requirements.txt, Dockerfile, or dependency specification is provided in the paper.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Experimental details in Appendix A.6 cover hyperparameters but do not provide step-by-step reproduction instructions; a reader would need to guess implementation details beyond what is stated.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are reported for any result; all tables report single-run normalized scores.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are used for any comparative claim; differences in percentage points are reported without p-values or hypothesis tests.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes are reported throughout in percentage point improvements (e.g., '+31.2pp for Llama BI+Dolci SGR vs. Dolci') with baseline context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The arithmetic ablation uses 200 problems (Appendix A.4) without justification or power analysis; benchmark evaluation sizes are determined by the benchmark, not chosen by the authors.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or run-to-run variability is reported for any experiment; all results are single-point estimates.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include: unpruned base models, standard SFT on open-source datasets (Alpaca, Dolci), and results from prior work (Lu et al. 2024) for direct comparison.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Comparisons use results from Lu et al. (2024) and contemporary pruning strategies (BI, Reverse Order) from 2024–2025 literature.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 conducts systematic single-layer ablations across all layers of 3 model families to identify which layers are sensitive to removal.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation uses 11 benchmarks: 7 classification (HellaSwag, PIQA, MMLU, WinoGrande, OBQA, ARC-E, ARC-C) and 4 generative (GSM8K, HumanEval+, MBPP+, XSUM), plus arithmetic accuracy and syntactic error rates.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Human evaluation is not applicable; all benchmarks use automated evaluation metrics appropriate for math, code, and classification tasks.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "All standard benchmarks (GSM8K, HumanEval+, MBPP+, classification benchmarks) have held-out test sets; finetuning is on training splits only.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by model family (4 models), pruning strategy (BI/Reverse/Iterative), training data (Alpaca/Dolci/SGR), and task type (classification vs. generative) throughout Tables 1–6.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.3 shows a concrete arithmetic mistake example (32×6=364) and Appendix A.5 shows a parenthesis mismatch error from a pruned model's code output.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.6.2 explicitly shows that even under ideal conditions (task-aligned pruning metric and training data), GSM8K performance cannot be fully recovered, demonstrating a fundamental limit.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact model versions are stated: Qwen-2.5-7B-Instruct, Llama-3.1-8B-Instruct, Mistral-7B-Instruct-v0.3, Gemma2-2B-It.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "The arithmetic probe prompt format is explicitly shown: 'Question: What is (7 + 5) - 6? Answer:'; standard benchmarks use established prompt formats via lm-evaluation-harness.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.6 reports: QLoRA with 4-bit NF4 quantization, learning rate 2×10^-4, batch size 8, 50 warmup steps, bf16, sequence length 8192, single A100 80GB GPU.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; the paper evaluates standard language model inference without agent frameworks.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "For arithmetic ablation, the EleutherAI/arithmetic dataset with single-digit, three-operations subset is specified; output space is restricted to digits 0–9; SGR generation procedure is described (prompts fed to unpruned base model).",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Raw model outputs (e.g., pruned model responses) are not released as part of the paper; only aggregate benchmark scores are reported.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "SGR data generation is described (unpruned base model generates 8 responses per prompt); benchmark data sources are identified; arithmetic subset selection criteria are stated.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; standard public benchmarks are used without recruitment.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The SGR pipeline is documented: (1) prompt extraction from open-source datasets, (2) response generation by unpruned base model, (3) SFT of pruned model on self-generated pairs; evaluation uses lm-evaluation-harness.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training data cutoffs for Qwen2.5, LLaMA3.1, Mistral, and Gemma2 are not stated or discussed anywhere in the paper.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Potential overlap between model pretraining data and evaluation benchmarks (GSM8K, HumanEval+, MMLU) is not discussed at all.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No mention of whether GSM8K, HumanEval+, or MMLU examples were available before training cutoffs; contamination is entirely unaddressed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost or latency measurements are reported; the paper focuses on recovery quality, not computational cost of the compressed models.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Only the hardware platform is mentioned (single A100 80GB for finetuning, NYU Abu Dhabi HPC cluster); no GPU-hours, FLOPs, or total compute budget is stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Generative reasoning tasks are substantially more sensitive to layer pruning than classification benchmarks, with classification retaining ~80%+ but generative tasks dropping to 14–32% retention.",
       "evidence": "Table 1 shows classification mean retention of 0.655–0.839 vs. generative retention of 0.143–0.327 across four model families at 25% pruning.",
       "supported": "strong"
     },
     {
       "claim": "Even single-layer removal can cause severe performance drops on GSM8K and HumanEval+, with sharp drops at specific layer positions varying by model family.",
       "evidence": "Figure 1 shows sharp drops in GSM8K and HumanEval+ performance for specific layers across Qwen, Llama, and Mistral, while XSUM remains largely stable.",
       "supported": "strong"
     },
     {
       "claim": "Layer pruning disrupts arithmetic computation capabilities, causing structural loss beyond text degeneration effects.",
       "evidence": "Figure 3 shows substantial drops in arithmetic logprob and accuracy after pruning specific Llama layers using a controlled evaluation that avoids multi-token generation demands.",
       "supported": "strong"
     },
     {
       "claim": "Text degeneration (repetition, Self-BLEU4) does not fully explain reasoning failures; performance drops occur even when text generation quality remains intact.",
       "evidence": "Section 4.1 shows that in Qwen and Llama, sharp performance drops on math/coding coincide with minimal degeneration metrics; Mistral shows the reverse pattern.",
       "supported": "strong"
     },
     {
       "claim": "Self-Generated Response (SGR) finetuning consistently outperforms finetuning on external open-source data for generative recovery, by up to 31.2 percentage points.",
       "evidence": "Table 2 shows SGR (BI + S.Dolci) achieves 63.4% generative retention for Llama vs. 32.2% for standard BI + Dolci, a +31.2pp improvement.",
       "supported": "strong"
     },
     {
       "claim": "Recovery of generative reasoning remains fundamentally limited relative to classification even under favorable conditions, with a persistent gap even after SGR finetuning.",
       "evidence": "Table 2 shows Llama SGR achieves 90.3% classification vs. 63.4% generative retention; Appendix A.6.2 shows full recovery is impossible even with task-aligned training data.",
       "supported": "strong"
     },
     {
       "claim": "Layer pruning is viable for generative tasks primarily at lower pruning ratios (~10–15%), with ~80% retention achievable at 10% pruning.",
       "evidence": "Figure 6 and Table 6 show ~85–90% generative retention at 2 layers pruned and declining performance beyond 10–15% pruning ratios for both Llama and Qwen.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "Layer pruning severely degrades generative reasoning (math, coding) while classification performance is largely preserved—a gap not explained by text degeneration alone. Pruning disrupts core algorithmic capabilities: arithmetic accuracy drops to ~34% average at 25% pruning, and parenthesis-matching errors spike at specific layers. Self-Generated Response (SGR) finetuning—using the unpruned base model's outputs as training targets—consistently outperforms open-source SFT by 20–31pp on generative benchmarks. Despite these gains, recovery is fundamentally limited: even under ideal task-aligned conditions, pruned models cannot fully recover baseline performance, suggesting that depth reduction irreversibly disrupts functional circuits required for multi-step reasoning.",
   "red_flags": [
     {
       "flag": "No error bars or variance",
       "detail": "All results are single-run normalized scores with no confidence intervals, standard deviations, or repeated experiment variance reported across any of the main tables."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (SGR vs. baseline SFT, classification vs. generative) are made without statistical hypothesis tests; differences could be within noise for individual benchmarks."
     },
     {
       "flag": "Contamination unaddressed",
       "detail": "Training cutoffs for all four evaluated models are not stated, and potential overlap between pretraining data and evaluation benchmarks (GSM8K, HumanEval+, MMLU) is never discussed."
     },
     {
       "flag": "Limited model scale",
       "detail": "All experiments use 2B–8B parameter instruct-tuned models; findings may not generalize to larger models (70B+) or base models, which the paper does not acknowledge as a limitation."
     },
     {
       "flag": "No competing interests statement",
       "detail": "The Impact Statement contains no declaration of competing interests, patents, or financial relationships."
     }
   ],
   "cited_papers": [
     {
       "title": "The Unreasonable Ineffectiveness of the Deeper Layers",
       "relevance": "Prior work showing classification robustness under layer pruning that this paper challenges for generative tasks."
     },
     {
       "title": "ShortGPT: Layers in Large Language Models Are More Redundant Than You Expect",
       "relevance": "Key prior work claiming layer redundancy; this paper refutes the generality of that claim for generative reasoning."
     },
     {
       "title": "Reassessing Layer Pruning in LLMs: New Insights and Methods",
       "relevance": "Direct comparison baseline whose results are incorporated into Table 1 and Table 3."
     },
     {
       "title": "Compact Language Models via Pruning and Knowledge Distillation",
       "relevance": "Large-scale recovery approach (Minitron) that this paper argues is impractical under constrained settings."
     },
     {
       "title": "LLM Pruning and Distillation in Practice: The Minitron Approach",
       "relevance": "Another knowledge distillation recovery baseline that requires pretraining-scale data."
     },
     {
       "title": "When Fewer Layers Break More Chains: Layer Pruning Harms Test-Time Scaling in LLMs",
       "relevance": "Related concurrent work on layer pruning failure modes for reasoning tasks."
     },
     {
       "title": "Layer Importance for Mathematical Reasoning Is Forged in Pre-Training and Invariant After Post-Training",
       "relevance": "Co-authored companion paper on mathematical reasoning sensitivity to layer removal."
     },
     {
       "title": "LLM Circuit Analyses Are Consistent Across Training and Scale",
       "relevance": "Cited to explain why functional circuits require pretraining-scale data to form and are hard to reconstruct post-pruning."
     },
     {
       "title": "Training Verifiers to Solve Math Word Problems (GSM8K)",
       "relevance": "Primary mathematical reasoning benchmark used throughout the evaluation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly informs practitioners doing model compression: provides clear guidance on when layer pruning is viable and when it will fail."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the optimistic literature on layer redundancy and classification-validated pruning by showing classification success is a poor proxy for generative capability."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk implications; the paper is about model compression quality, not alignment or misuse."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Modest conflict angle: exposes limits of a popular technique endorsed by prior work, but not a heated public controversy."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released on GitHub but running the experiments requires an A100 GPU and significant compute, limiting casual reproducibility."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "NYU Abu Dhabi is a recognized institution; the paper uses prominent models (LLaMA, Qwen, Mistral, Gemma) but is not from a major AI lab."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }