ai-research-survey

scan.json (26900B)

---

 {
   "paper": {
     "title": "Code Less, Align More: Efficient LLM Fine-tuning for Code Generation with Data Pruning",
     "authors": ["Yun-Da Tsai", "Mingjie Liu", "Haoxing Ren"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2407.05040"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The paper references public datasets but does not release its own implementation."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available datasets: Magicoder-OSS-Instruct-75K (MIT License) and Magicoder-Evol-Instruct-110K (Apache-2.0 License), both hosted on HuggingFace with URLs provided in Section 4.2. The evaluation benchmarks (HumanEval, MBPP, EvalPlus) are also public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 16 NVIDIA A100-80GB GPUs, PyTorch DDP, vLLM framework, and bf16 dtype (Section 4.2), but does not provide a requirements.txt, Dockerfile, or detailed dependency list with library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README with commands are provided. Algorithm 1 gives pseudocode for the pruning strategy, and training hyperparameters are stated (Section 4.1-4.2), but there are no specific scripts or commands a researcher could follow to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions standard deviation in the text ('we report the average and standard deviation' in Section 4.1), the main results in Table 1 report only averaged pass@1 values without confidence intervals or error bars. Figures 2 and 4 show shaded regions suggesting variance bands, but no CI notation or explicit ± values are given in tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (e.g., 2.7% on HumanEval, 3.5% on MBPP) over full dataset training but does not report any statistical significance tests (no p-values, t-tests, or bootstrap tests). Claims of outperformance are based solely on comparing averaged point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout. For example, Section 4.5: '2.7% on HumanEval and 3.5% on MBPP compared to training with the full dataset' and '3.9% on HumanEval and 1.5% on MBPP' degradation at 10% data. Table 1 provides raw scores for all methods, allowing readers to compute effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 3 experimental runs per configuration (Section 4.1) but does not justify why 3 runs is sufficient. The Limitations section acknowledges 'Due to computational constraints, we only performed three runs' and that this 'may not fully capture the variability,' but no power analysis or formal justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 states 'we repeat each experiment 3 times and report the average and standard deviation,' and Figures 2, 4, 5 show shaded variance bands. However, Tables 1, 2, and 3 report only point estimates without any standard deviation values. The main numerical results lack explicit spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines: nocluster-random (no clustering, random selection), GPT-3.5 Turbo, GPT-4 Turbo, DeepSeek-Coder-Base/Instruct, Magicoder-DS, and MagicoderS-DS (Table 1). Ablation baselines include different clustering algorithms and pruning metrics."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4 Turbo, DeepSeek-Coder (2024), MagicoderS-DS (2023), which are contemporary models at the time of writing. The paper uses state-of-the-art code LLMs as reference points."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 presents four ablation studies: (1) clustering algorithms (KMeans, Agglomerative, HDBSCAN, no clustering — Figure 4), (2) pruning metrics (density, diversity, random — Figure 5), (3) effect of PCA (Table 2), and (4) embedding inputs (instruction, code, or both — Table 3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports pass@1 on four benchmarks: HumanEval, HumanEval+, MBPP, and MBPP+ (Table 1). EvalPlus provides augmented test suites (80x and 35x more tests) for more rigorous evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via pass@1 on test cases from HumanEval/MBPP benchmarks. Given that the paper claims improvements in 'overall quality code generation,' human assessment of code quality beyond functional correctness would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation benchmarks (HumanEval, MBPP, and their EvalPlus variants) are separate from the training data (Magicoder-OSS-Instruct-75K and Magicoder-Evol-Instruct-110K). The training and evaluation datasets are distinct."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across four individual benchmarks (HumanEval, HumanEval+, MBPP, MBPP+) in Table 1 and across different compression ratios (10%-90%) in Figures 2-5. This shows how performance varies across settings."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 in the Appendix shows an example of a removed data sample (an outlier that is a product management prompt, not a coding task). Section 5.3 discusses the negative impact of PCA on HumanEval performance (4.3% degradation). The Limitations section acknowledges variability issues."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: PCA degrades HumanEval performance by 4.3% (Table 2, Section 5.3); pruning metrics show limited benefit beyond clustering alone (Section 5.2, Figure 5); aggressive pruning below 10% degrades HumanEval performance significantly (Figure 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims: (1) benchmark performance preserved at 10% data — supported by Table 1 showing 70.4% vs 74.3% on HumanEval; (2) consistent improvements through moderate pruning — supported by 90% compression row showing 77.0% vs 74.3%; (3) reduced computational resources — supported by reduced training tokens (192M vs 234M). Claims are hedged appropriately ('largely preserved,' 'consistent improvements')."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about pruning improving performance (e.g., 'moderate pruning of the training data... leading to improvements'). These are supported by controlled ablation studies in Section 5 where single variables are manipulated (clustering algorithm, pruning metric, PCA, embedding input) while holding others constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract make broad claims about 'LLM Fine-tuning for Code Generation' but all experiments use a single base model (DeepSeek-Coder-Base 6.7B), two synthetic training datasets, and Python-only benchmarks. The Potential Risks section notes 'This study focus exclusively on English prompts for Python code generation' but the title and abstract do not reflect these scope limitations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why pruning improves performance. For instance, the improvement could be due to removing noisy/incorrect synthetic data rather than better sample diversity, or the batch size change at small data sizes could explain some results. No alternative explanations or confounds are considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'DeepSeek-Coder-Base 6.7B' and 'OpenAI text-embedding-ada-002' but does not provide specific version snapshots or API dates. For the compared models (GPT-3.5 Turbo, GPT-4 Turbo), no version identifiers are given — results are taken from prior work without specifying which model snapshot."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper fine-tunes a base model on instruction-code pairs and evaluates using EvalPlus scripts with existing benchmarks. There is no custom prompting of LLM APIs as part of the method — the approach is about data pruning for fine-tuning, not prompt engineering."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 provides detailed hyperparameters: learning rate 5e-5, 15 warmup steps, linear scheduler, Adam optimizer, batch size 512 (or 32 for small datasets), 2 epochs, sequence length 4096. Section 4.3 specifies inference settings: bf16 dtype, tensor parallel size 2, max length 4096, greedy decoding. Section 4.4 reports PCA dimension of 10."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a data pruning pipeline for fine-tuning, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 and Algorithm 1 document the full preprocessing pipeline: embedding via text-embedding-ada-002, PCA dimensionality reduction to 10 dimensions, clustering (HDBSCAN/KMeans/Agglomerative), and pruning metric application. Section 4.4 provides additional details including Scott's Rule for KDE bandwidth and the Elbow method for cluster selection."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section (after Section 6) discusses two specific limitations: (1) only 3 runs due to computational constraints may not fully capture variability, and (2) batch size changes for small datasets and unexplored optimal hyperparameters."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: 'Due to computational constraints, we only performed three runs and averaged the results... it may not fully capture the variability and could lead to less accurate conclusions.' It also notes the batch size change from 512 to 32 for small datasets could affect comparisons. The Potential Risks section discusses language and safety limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Potential Risks section explicitly states: 'This study focus exclusively on English prompts for Python code generation, thus prompts in other languages might not produce accurate or functional code.' The Limitations section notes that optimal hyperparameters for different data sizes are left as future work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the training datasets are publicly available on HuggingFace, the pruned subsets, cluster assignments, and per-run experimental results are not released. There is no way to verify which specific data points were selected by the pruning algorithm."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the training data: Magicoder-OSS-Instruct-75K and Magicoder-Evol-Instruct-110K, totaling 185K entries. These are synthetic code datasets with clearly stated licenses and HuggingFace URLs. Evaluation datasets (HumanEval with 164 problems, MBPP with 1401 problems) are described in Section 4.3."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data consists of standard public benchmarks and synthetic code datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 and Section 3 document the full pipeline: raw instruction-code pairs → embedding → PCA reduction → clustering → pruning metric scoring → probabilistic selection. Section 4.4 provides implementation details including specific parameter choices."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. All three authors are affiliated with NVIDIA, but no explicit funding disclosure is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are clearly listed with their NVIDIA affiliation and email addresses on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are NVIDIA employees. NVIDIA has a direct commercial interest in efficient LLM training methods, as it sells the GPU hardware (A100s) used in the experiments and competes in the AI/LLM market. The funder (NVIDIA as employer) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper fine-tunes DeepSeek-Coder-Base 6.7B but does not state its training data cutoff date. This is relevant because HumanEval (published 2021) and MBPP (published 2021) could have been in the base model's pretraining data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between DeepSeek-Coder-Base's pretraining data and the HumanEval/MBPP benchmarks. Since the base model was likely trained on public code including these benchmarks, this is a relevant concern not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) were published well before DeepSeek-Coder's training. The paper does not address whether these benchmarks were in the base model's training data, which could inflate baseline performance and affect the measured impact of pruning."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report inference cost or latency for the pruning pipeline itself (embedding + PCA + clustering + pruning). Table 2 reports runtime for KMeans with/without PCA (1307 sec vs 183 sec) but this is only for one configuration at 50% compression, and training/inference costs for the fine-tuned models are not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 16 NVIDIA A100-80GB GPUs for training (Section 4.2) and reports training token counts (Table 1), but does not state total GPU hours, wall-clock training time, or API costs for the embedding step using text-embedding-ada-002."
       }
     }
   },
   "claims": [
     {
       "claim": "Benchmark performance can be largely preserved by training on only 10% of the data, with slight degradation of 3.9% on HumanEval and 1.5% on MBPP compared to using all data.",
       "evidence": "Table 1: Ours (10%) achieves 70.4% on HumanEval vs 74.3% full data, and 73.0% on MBPP vs 74.5% full data. Section 4.5 discusses these results.",
       "supported": "strong"
     },
     {
       "claim": "Moderate pruning (90% retention) consistently improves benchmark performance, with improvements of up to 2.7% on HumanEval and 3.5% on MBPP compared to full dataset training.",
       "evidence": "Table 1: Ours (90%) achieves 77.0% on HumanEval vs 74.3% full data (+2.7%), and 76.9% on MBPP vs 74.5% (+2.4%, though 3.5% is claimed for MBPP). Results averaged over 3 runs.",
       "supported": "moderate"
     },
     {
       "claim": "HDBSCAN-diversity method consistently outperforms the nocluster-random baseline across different compression ratios.",
       "evidence": "Figure 2 shows HDBSCAN-diversity above nocluster-random across all compression ratios on both MBPP and HumanEval benchmarks. Section 4.5 discusses this.",
       "supported": "moderate"
     },
     {
       "claim": "Clustering algorithm choice is critical, while pruning metrics have limited impact.",
       "evidence": "Section 5.1 (Figure 4) shows clustering algorithms improve over no-clustering baseline. Section 5.2 (Figure 5) shows diversity metric has only 'marginal advantage' and 'its benefit may be limited.'",
       "supported": "moderate"
     },
     {
       "claim": "Even with just 1% of the data (~700 samples), the method achieves a 4.1% improvement over the base model on MBPP.",
       "evidence": "Figure 3 and Table 1: Ours (1%) achieves 74.3% on MBPP vs base model 70.2%, a 4.1% improvement. Section 4.5 discusses this.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Data pruning via HDBSCAN clustering with diversity-based selection can maintain or improve code generation performance while significantly reducing fine-tuning data. Training on 90% less data (10% retention) preserves most benchmark performance, with only 3.9% degradation on HumanEval and 1.5% on MBPP. Moderate pruning (10-20% compression) actually improves performance over full-data training by up to 2.7% on HumanEval and 3.5% on MBPP. Ablation studies show the clustering algorithm is the critical component, while the specific pruning metric has limited additional impact.",
   "red_flags": [
     {
       "flag": "No significance tests with small effect sizes",
       "detail": "Improvements of 2-3.5% are claimed over 3 runs without any statistical significance testing. Given the acknowledged randomness in clustering and training, these differences may not be statistically significant."
     },
     {
       "flag": "Single base model tested",
       "detail": "All experiments use only DeepSeek-Coder-Base 6.7B. Claims about 'LLM fine-tuning for code generation' are not tested on other model families or sizes, limiting generalizability."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "HumanEval and MBPP were published in 2021, likely before DeepSeek-Coder's training cutoff. No discussion of whether the base model already memorized these benchmarks, which would affect the interpretation of pruning effects."
     },
     {
       "flag": "NVIDIA employees evaluating compute efficiency method",
       "detail": "All authors are NVIDIA employees presenting a method that reduces training compute while maintaining performance. NVIDIA has a commercial interest in demonstrating efficient GPU utilization. No conflict of interest statement is provided."
     },
     {
       "flag": "Variance not reported in tables",
       "detail": "Despite claiming 3-run averaging with standard deviation, Tables 1, 2, and 3 report only point estimates. The reader cannot assess result stability from the main results."
     },
     {
       "flag": "Batch size confound",
       "detail": "The batch size is changed from 512 to 32 for datasets below 10% of original size. This confounds the comparison of pruning levels, as the batch size change could independently affect training outcomes."
     }
   ],
   "cited_papers": [
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming – The Rise of Code Intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "relevance": "Major open-source code LLM used as the base model; directly relevant to LLM code generation capabilities."
     },
     {
       "title": "Magicoder: Source Code is All You Need",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu", "Yifeng Ding", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2312.02120",
       "relevance": "Key baseline for code LLM fine-tuning with synthetic data; introduces OSS-Instruct technique used in this paper's training data."
     },
     {
       "title": "WizardCoder: Empowering Code Large Language Models with Evol-Instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2024,
       "relevance": "Baseline code LLM that uses Evol-Instruct for synthetic data generation; directly relevant to code generation quality."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, the primary evaluation metric used in code generation research."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Introduces EvalPlus with augmented test suites (HumanEval+, MBPP+); directly relevant to code generation evaluation methodology."
     },
     {
       "title": "LIMA: Less is More for Alignment",
       "authors": ["Chunting Zhou", "Pengfei Liu", "Puxin Xu"],
       "year": 2023,
       "relevance": "Introduces the Superficial Alignment Hypothesis suggesting minimal fine-tuning data suffices; foundational to the data pruning motivation."
     },
     {
       "title": "LESS: Selecting Influential Data for Targeted Instruction Tuning",
       "authors": ["Mengzhou Xia", "Sadhika Malladi", "Suchin Gururangan", "Sanjeev Arora", "Danqi Chen"],
       "year": 2024,
       "arxiv_id": "2402.04333",
       "relevance": "Data selection method for instruction tuning using gradient-based influence metrics; directly relevant to efficient LLM training."
     },
     {
       "title": "Beyond Neural Scaling Laws: Beating Power Law Scaling via Data Pruning",
       "authors": ["Ben Sorscher", "Robert Geirhos", "Shashank Shekhar", "Surya Ganguli", "Ari Morcos"],
       "year": 2022,
       "relevance": "Foundational work on data pruning demonstrating that careful data selection can beat scaling laws; directly relevant methodology."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code LLM built on LLaMA2; relevant to the landscape of code generation models."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM referenced for scaling laws and as a benchmark comparison point."
     },
     {
       "title": "AlpaGasus: Training a Better Alpaca with Fewer Data",
       "authors": ["Lichang Chen", "Shiyang Li", "Jun Yan"],
       "year": 2024,
       "relevance": "Data filtering method for instruction tuning using quality metrics; directly relevant to efficient fine-tuning approaches."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["Edward J Hu", "Yelong Shen", "Phillip Wallis"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning method; relevant to efficient LLM training approaches."
     }
   ]
 }