ai-research-survey

scan.json (25435B)

---

 {
   "paper": {
     "title": "On the Limits of Layer Pruning for Generative Reasoning in LLMs",
     "authors": ["Safal Shrestha", "Anubhav Shrestha", "Aadim Nepal", "Minwu Kim", "Keith Ross"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.01997"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Layer pruning preserves classification performance (80%+ retention at 25% pruning) but severely degrades generative reasoning tasks. The degradation stems from disruption of core algorithmic capabilities (arithmetic, parenthesis tracking), not just surface-level text degeneration. Supervised finetuning with Self-Generated Responses (SGR) improves recovery by 20-30pp over standard finetuning but cannot fully restore generative reasoning, especially at aggressive pruning ratios. Moderate pruning (≤10-15%) with SGR offers the best practical trade-off.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/safal312/on-the-limits-of-layer-pruning (Section 1, footnote 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Data and models available on HuggingFace: https://huggingface.co/collections/safal312/on-the-limits-of-generative-reasoning-in-llms (Section 1, footnote 2). All benchmarks used are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'single A100 80GB GPU' and QLoRA settings but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A GitHub link exists but the paper itself does not describe how to reproduce experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (retention ratios). No confidence intervals or error bars are shown in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (SGR outperforms standard finetuning, classification retains more than generative) but no statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentage point differences with baseline context throughout, e.g., 'gain of +25.9 percentage points relative to finetuning on Alpaca alone' (Section 5.1), and retention ratios normalize against baseline performance."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of benchmarks, models, or the 200 arithmetic problems used in the ablation study."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines included: unpruned model, standard SFT with open-source data (Alpaca, Dolci), and results from Lu et al. (2024) are compared against the proposed SGR approach."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent pruning strategies (BI, Reverse Order from Men et al. 2025, Lu et al. 2024) and recent model families (Llama 3.1, Qwen 2.5, Gemma 2)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation: single-layer removal analysis (Section 4), comparison across pruning strategies (BI, Reverse, Iterative), training data sources (Alpaca vs Dolci, standard vs SGR), and pruning ratios (Section 5.3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics across classification (HellaSwag, PIQA, MMLU, WinoGrande, OBQA, ARC-E, ARC-C) and generative (GSM8K, HumanEval+, MBPP+, XSUM) benchmarks, plus text degeneration metrics (4-gram repetition, Self-BLEU4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via benchmarks and metrics. Manual inspection is mentioned briefly for qualitative examples but not as systematic evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets are used (GSM8K test set mentioned explicitly in A.6.2, HumanEval+/MBPP+ are standard held-out sets). Finetuning uses separate training data (Alpaca, Dolci)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-benchmark breakdowns are provided in all tables (Tables 1-6), showing individual results for each classification and generative benchmark rather than only aggregate scores."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Detailed failure analysis in Sections 4.1-4.3: text degeneration, arithmetic errors (Appendix A.3 example), parenthesis tracking failures (Appendix A.5 example), and syntactic error distributions (Figure 4)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "A central finding is negative: SGR finetuning cannot fully recover generative reasoning. Section A.6.2 reports an upper-bound experiment that still fails. Iterative pruning does not consistently outperform simpler strategies."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about classification vs generative gap are supported by Tables 1-3; 'up to 90% baseline performance' on classification is shown in Table 2; '20-30 percentage points' gains on generative are shown across multiple ablations."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('pruning disrupts arithmetic capability') are supported by controlled single-layer ablation experiments isolating the effect of individual layer removal (Figure 3). The controlled evaluation with logprob measurement (Section 4.2) strengthens causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests across 4 model families and explicitly notes model-specific differences (e.g., 'Qwen exhibits less layer redundancy'). The title appropriately focuses on 'layer pruning' not all compression. Discussion notes limitations 'under realistic post-training constraints.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly rules out text degeneration as the sole explanation for generative failure, showing 'degeneration alone does not fully account for failures.' Section 7 discusses alternative theories about shallow vs deep subnetworks (Petty et al., Telgarsky)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes between proxy metrics (benchmark accuracy, retention ratios) and the underlying capabilities (arithmetic competence, syntactic ability, reasoning). Section 4.2 explicitly creates a controlled evaluation to isolate arithmetic from generation demands."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are named: Qwen-2.5-7B-Instruct, Llama-3.1-8B-Instruct, Mistral-7B-Instruct-v0.3, Gemma2-2B-It. These are sufficiently specific versioned model names."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The arithmetic evaluation prompt format is provided: 'Question: What is (7 + 5) - 6? Answer:' (Section 4.2). The paper primarily uses standard benchmark evaluation harnesses (lm-evaluation-harness) with standard prompting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "QLoRA hyperparameters reported in A.6.1: '4-bit NF4 quantization, learning rate 2×10⁻⁴, batch size 8, constant learning rate with 50 warmup steps, bf16 training, sequence length 8192.' Training epochs specified per dataset."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a model compression study using standard evaluation frameworks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "SGR data generation process is described: 'we pass only the prompts from the open-source datasets to the base model and use its generated outputs' (Section 5). Alpaca-cleaned and Dolci datasets identified. For upper-bound experiment, 'eight responses from the unpruned Qwen model' per example (A.6.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion & Conclusion) contains substantive discussion of limitations, including the persistent gap in recovery and the constraints of post-training settings."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific limitations discussed: recovery may only work 'under constrained settings studied here' (Section 7), QLoRA vs full finetuning comparison (A.6.3), model-specific behavior (Qwen vs Llama redundancy differences), and acknowledged that 'aggressive pruning may be impractical.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope: 'Rather than proposing a new pruning algorithm, our goal is to characterize the limits' (Section 1). It bounds results to 'realistic post-training constraints, without access to pretraining-scale data or compute.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Code and models are released on GitHub and HuggingFace, which should include the experimental outputs. All benchmarks used are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are clearly described: standard public benchmarks (GSM8K, HumanEval+, MBPP+, XSUM, classification benchmarks), plus Alpaca-cleaned and Dolci datasets for finetuning. Arithmetic evaluation uses EleutherAI/arithmetic dataset (A.4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: pruning strategy → optional finetuning → evaluation on benchmarks. SGR data generation process is described. The iterative pruning algorithm is formally specified (Algorithm 1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section: 'submitted in part by the NYU Abu Dhabi Center for Artificial Intelligence and Robotics, funded by Tamkeen under the Research Institute Award CG010.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed as Department of Computer Science, New York University Abu Dhabi. No conflict with evaluated products (they evaluate open-source models from Meta, Google, Alibaba, Mistral)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funder (Tamkeen/NYU Abu Dhabi) is an academic research institute with no financial stake in the outcome of layer pruning evaluations."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four models evaluated, despite using benchmarks like GSM8K and HumanEval that could be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems (GSM8K, HumanEval+) appeared in the training data of the evaluated models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) and HumanEval (2021) are well-known to be in many models' training data. All models tested were released in 2024-2025, well after these benchmarks. No contamination discussion."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency numbers are reported despite evaluating model compression methods where efficiency is a core motivation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('single A100 80GB GPU') but total GPU hours, training time, or computational budget are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple seed experiments reported. All results appear to be single-run numbers."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The QLoRA settings appear fixed but no justification for these choices is given."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Multiple configurations are compared (different pruning strategies, datasets) but the selection of reported configurations is not justified beyond standard choices from prior work."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes SGR and compares against baselines without acknowledging self-comparison bias. However, many baselines are sourced from Lu et al. (2024), partially mitigating this."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis despite SGR requiring generating responses from the full unpruned model, which is an additional compute cost not compared against distillation approaches."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses construct validity: Section 4.2 argues that standard generative benchmarks 'can obscure true abilities due to auxiliary demands' and designs controlled evaluations to isolate arithmetic competence. Section 4 discusses what benchmarks actually measure."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly on benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. GSM8K and HumanEval were published years before the models' training, creating clear temporal leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. HumanEval+ provides function signatures as prefixes which could interact with memorized solutions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Generative reasoning tasks are substantially more sensitive to layer pruning than classification benchmarks, with even single-layer removal causing severe degradation.",
       "evidence": "Table 1 shows classification retention of 65-84% vs generative retention of 14-33% at 25% pruning across 4 models. Figure 1 shows sharp drops from single-layer removal on GSM8K and HumanEval+.",
       "supported": "strong"
     },
     {
       "claim": "Layer pruning disrupts core algorithmic capabilities (arithmetic computation, parenthesis tracking), not just surface-level text generation.",
       "evidence": "Section 4.2: controlled arithmetic evaluation shows accuracy dropping to 34.3% after pruning (Figure 7). Figure 3 shows logprob degradation. Figure 4 shows syntactic error spikes. Section 4.1 demonstrates degeneration alone doesn't explain failures.",
       "supported": "strong"
     },
     {
       "claim": "SGR finetuning achieves gains of up to 20-30 percentage points on generative benchmarks compared to standard finetuning.",
       "evidence": "Table 2: Llama BI + S.Dolci achieves 63.4% generative retention vs 32.2% for BI + Dolci (+31.2pp). Similar gains observed across models (Table 5).",
       "supported": "strong"
     },
     {
       "claim": "Recovery for generative reasoning remains fundamentally limited even after SGR finetuning.",
       "evidence": "Figure 7: arithmetic accuracy recovers to only 58% from 34.3% (base: 77.3%). Section A.6.2 upper-bound experiment with task-aligned full SFT still cannot recover baseline. Persistent gap between classification (90%) and generative (63%) retention.",
       "supported": "strong"
     },
     {
       "claim": "Moderate pruning ratios (≤10-15%) with SGR offer a practical trade-off for preserving generative reasoning.",
       "evidence": "Figure 6 and Table 6: at 2 layers removed, ~85-90% retention; at ~10%, ~80% for BI/Iterative. Beyond 15%, steady decline.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or uncertainty reporting",
       "detail": "All results are single-run point estimates with no error bars, standard deviations, or confidence intervals. Given that pruning and finetuning outcomes can be sensitive to random seeds, the stability of results is unknown."
     },
     {
       "flag": "No contamination analysis",
       "detail": "GSM8K and HumanEval were published in 2021. All evaluated models (Llama 3.1, Qwen 2.5, etc.) were trained on data collected well after 2021 and likely include these benchmarks. Performance differences between pruned and unpruned models could partly reflect memorization rather than reasoning capability."
     },
     {
       "flag": "Missing compute cost comparison",
       "detail": "SGR requires running inference with the full unpruned model to generate training targets, but this cost is never quantified or compared against the distillation approaches the paper argues are too expensive."
     }
   ],
   "cited_papers": [
     {
       "title": "The unreasonable ineffectiveness of the deeper layers",
       "authors": ["Andrei Gromov", "Kushal Tirumala", "Hassan Shapourian", "Paolo Glorioso", "Daniel A. Roberts"],
       "year": 2024,
       "arxiv_id": "2403.17887",
       "relevance": "Key prior work on layer redundancy in LLMs, finding that deep layers can be removed with minimal classification impact."
     },
     {
       "title": "Shortened LLaMA: Depth Pruning for Large Language Models with Comparison of Retraining Methods",
       "authors": ["Bo-Kyeong Kim"],
       "year": 2024,
       "arxiv_id": "2402.02834",
       "relevance": "Comparative study of retraining methods after depth pruning, directly relevant to compression and efficiency."
     },
     {
       "title": "ShortGPT: Layers in Large Language Models Are More Redundant Than You Expect",
       "authors": ["Xin Men"],
       "year": 2025,
       "relevance": "Demonstrates layer redundancy using Block Influence metric, one of the main pruning strategies evaluated."
     },
     {
       "title": "Compact Language Models via Pruning and Knowledge Distillation",
       "authors": ["Saurav Muralidharan"],
       "year": 2024,
       "relevance": "Large-scale pruning and distillation approach (Minitron), represents the compute-heavy alternative to SGR."
     },
     {
       "title": "LLM Pruning and Distillation in Practice: The Minitron Approach",
       "authors": ["Sharath Turuvekere Sreenivas"],
       "year": 2024,
       "arxiv_id": "2408.11796",
       "relevance": "Practical pruning and distillation pipeline for LLMs, baseline for recovery methods."
     },
     {
       "title": "Reassessing Layer Pruning in LLMs: New Insights and Methods",
       "authors": ["Yifei Lu"],
       "year": 2024,
       "arxiv_id": "2411.15558",
       "relevance": "Source of several baseline results used in this paper; reassesses layer pruning methods."
     },
     {
       "title": "When Fewer Layers Break More Chains: Layer Pruning Harms Test-Time Scaling in LLMs",
       "authors": ["Kaiqi Wang"],
       "year": 2025,
       "arxiv_id": "2510.22228",
       "relevance": "Studies how layer pruning affects chain-of-thought reasoning and test-time scaling."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws, foundational work on compute-optimal training relevant to model efficiency."
     },
     {
       "title": "SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot",
       "authors": ["Elias Frantar", "Dan Alistarh"],
       "year": 2023,
       "arxiv_id": "2301.00774",
       "relevance": "One-shot sparsification method for LLMs, complementary compression technique to layer pruning."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Questions emergent abilities and evaluation metrics — relevant to how benchmark scores can be misleading."
     }
   ]
 }