ai-research-survey

scan.json (26570B)

---

 {
   "paper": {
     "title": "Genetic Instruct: Scaling up Synthetic Generation of Coding Instructions for Large Language Models",
     "authors": [
       "Somshubra Majumdar",
       "Vahid Noroozi",
       "Mehrzad Samadi",
       "Sean Narenthiran",
       "Aleksander Ficek",
       "Wasi Uddin Ahmad",
       "Jocelyn Huang",
       "Jagadeesh Balam",
       "Boris Ginsburg"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2407.21077",
     "doi": "10.48550/arXiv.2407.21077"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Genetic-Instruct generates synthetic coding instructions using evolutionary crossover and mutation operations starting from 512 seed instructions, producing 7.5M instruction-code pairs. Models fine-tuned on this data achieve 69.7% average accuracy across HumanEval/MBPP benchmarks, outperforming alternative synthetic generation methods (best baseline 66.8%) and publicly available datasets. Combining mutation and crossover yields better results than either alone (68.0% vs 66.8% and 66.6%). Performance shows diminishing returns beyond ~6M samples, and even smaller generator models (Qwen-7B) can produce competitive synthetic data.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository is mentioned. The paper releases the dataset on HuggingFace but provides no code for the Genetic-Instruct pipeline itself."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The 7.5M synthetic dataset is publicly released at https://huggingface.co/datasets/nvidia/OpenCodeGeneticInstruct, mentioned in Section 1 and the conclusion."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions NeMo framework, NeMo Aligner, vLLM, BF16 precision, and tensor parallelism (Section 4.1) but provides no requirements.txt, Dockerfile, or specific library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The algorithm is described in Section 3 and Algorithm 1, but there are no runnable scripts or README-style instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates (e.g., '69.7%') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'significant improvement' and that models 'consistently outperform' baselines, but no statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-3 report absolute accuracy numbers for all methods, providing baseline context. E.g., Genetic-Instruct achieves 69.7% vs best public dataset at 62.9% (Table 1), allowing readers to compute effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 512 seed instructions, why 4M or 7.5M samples, or why these particular benchmark sizes. Values are stated without justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance across runs, or any spread measure is reported. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against WizardCoder, Self-Instruct, OSS-Instruct, INVERSE-INSTRUCT, and several public datasets (Code Parrot Apps, TACO, OpenCoder, Code Alpaca), plus Llama 3.1 8B Instruct."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include WizardCoder (2024), OSS-Instruct/Magicoder (2024), INVERSE-INSTRUCT (2024), and OpenCoder (2024) — all contemporary with the paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 ablates crossover-only vs mutation-only vs combined. Table 3 ablates the effect of different generator models (Mixtral-8x22B, Mixtral-8x7B, Qwen-32B, Qwen-7B)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on four benchmarks: HumanEval, MBPP, HumanEval+, and MBPP+ (Tables 1-3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated instructions or code quality. All evaluation is automated via benchmark pass rates. Human evaluation of the quality/diversity of generated synthetic instructions would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses standard held-out benchmarks (HumanEval, MBPP, HumanEval+, MBPP+) that are separate from the synthetic training data, with decontamination applied (Section 3.6)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-3 provide per-benchmark breakdowns (HumanEval, MBPP, HumanEval+, MBPP+) rather than just aggregate averages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of failure cases — what types of instructions Genetic-Instruct struggles to generate, what kinds of code solutions fail the Judge-LLM, or where fine-tuned models break down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 reports diminishing returns beyond ~6M samples, showing the limits of scaling. The paper also reports that INVERSE-INSTRUCT performs poorly (41.1% average), and that mutation-only slightly outperforms the combined approach on HumanEval."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'significant improvement,' 'highly parallelizable,' and 'effective even with small seed data and weaker generator models' are supported by Table 1 (improvement over baselines), Section 3.5 (parallelization), and Table 3 (Qwen-7B competitive with Qwen-32B)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 2) uses controlled single-variable manipulation — same base model, same data size, same generator, varying only the algorithm. This is adequate for causal claims about the contribution of mutation and crossover operations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Coding Instructions for Large Language Models' broadly, but evaluation is exclusively on Python benchmarks (HumanEval/MBPP). Section 4 states 'our evaluation focuses exclusively on Python coding benchmarks' but the abstract and title do not bound the claims to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. Could the gains be from data volume alone? Are the baseline re-implementations optimal? Could different seed data change the ranking? None of these are addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass rates on coding benchmarks and frames results as 'code generation capability' — the benchmarks directly test code generation, so the proxy closely matches the claim. No broader unsupported framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are provided: Llama3.1-8B-Base, Mixtral-8x22B, Mixtral-8x7B, Qwen2.5-7B-Base, Qwen-32B, Qwen-7B, Meta-Llama-3-70B-Instruct. These are sufficiently specific for open-source models with distinct releases."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendices A-F: mutation prompts (Figure 3), crossover prompt (Figure 4), code generation prompt (Figure 5), fitness/judge prompt (Figure 6), decontamination prompt (Figure 7), and evaluation prompts (Figures 8-9)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports: temperature 1.2 for Instructor-LLM, 1.0 for Coder/Judge-LLM, learning rate 5e-6 decaying to 5e-7, cosine annealing, 3 epochs, nucleus sampling, max sequence length 1024, batch sizes Bm=100, Bc=10, mutation probability Mp=0.5, 20 parallel colonies."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a data generation pipeline with sequential LLM calls (Instructor → Coder → Judge), not an agent with tools, retry logic, or memory."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full pipeline: seed selection from Tiger-Leetcode (512 samples), crossover/mutation operations, AST validation of generated code (Section 3.3), Judge-LLM filtering (Section 3.4), and decontamination (Section 3.6)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. The paper proceeds from experiments (Section 4) directly to conclusion (Section 5) with no discussion of limitations or threats to validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 briefly states 'our evaluation focuses exclusively on Python coding benchmarks' but does not systematically state scope boundaries — what was not tested, what populations are excluded, or what claims are not being made."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full 7.5M synthetic dataset is released on HuggingFace (nvidia/OpenCodeGeneticInstruct), enabling independent verification of the training data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full data generation procedure: starting from 512 Tiger-Leetcode seeds, applying crossover/mutation via Instructor-LLM, generating code via Coder-LLM, filtering via AST checks and Judge-LLM, and decontamination."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets (Tiger-Leetcode seeds, Stack v2 for baselines)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 and Section 3 detail each pipeline stage: seed selection → crossover/mutation → instruction generation → code generation → AST validation → Judge-LLM filtering → aggregation → decontamination."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section. All authors are NVIDIA employees but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All nine authors are clearly identified as NVIDIA employees with @nvidia.com email addresses listed in the header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "NVIDIA has commercial interest in compute-intensive methods succeeding — synthetic data generation at scale requires significant GPU resources. The company benefits from demonstrating the value of large-scale GPU-based data generation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosures appear in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for the base models (Llama 3.1, Mixtral, Qwen) are not stated, making it impossible to assess whether the models' pre-training data included benchmark solutions."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 describes a decontamination process for their synthetic data: embedding-based similarity search + LLM paraphrase detection against all benchmark datasets, with positional bias control."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 applies a concrete decontamination methodology (Yang et al., 2023) using Sentence Transformer similarity search and Meta-Llama-3-70B-Instruct paraphrase detection, with dual-direction matching to control positional bias."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs, or wall-clock time is reported for the generation pipeline despite producing 7.5M samples using multiple LLMs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total GPU hours, training time, or hardware specification is provided. The paper mentions tensor parallelism and vLLM but does not quantify the total computational budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. All results appear to be single-run numbers."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from a single run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameter values are reported (Section 4.1) but no search budget is stated. Batch sizes are justified as 'based on our observation' without specifying how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the final configuration was selected. Hyperparameters appear chosen but the selection process is not described."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors re-implement all baselines (WizardCoder, Self-Instruct, OSS-Instruct, INVERSE-INSTRUCT) themselves. While they justify this for fairness ('same generator model, seed population, base model'), they do not acknowledge the inherent bias of implementing competing methods (Lucic et al., 2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "All methods generate the same number of samples (4M), but the compute cost per method likely differs (mutation requires per-sample LLM calls, crossover batches instructions). This is not analyzed or compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval/MBPP actually measure 'code generation capability' broadly, or their known limitations (e.g., simple function-level problems, limited language coverage)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — the evaluation is direct model fine-tuning and benchmark pass rates."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) were published years before Llama 3.1 and Mixtral were trained. The base models may have seen solutions during pre-training, but this temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides hints not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether the synthetically generated instructions share structural similarities with benchmark problems, despite using coding questions as seeds."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 applies a concrete leakage detection method: embedding-based similarity search using Sentence Transformer followed by LLM paraphrase detection (Meta-Llama-3-70B-Instruct) with positional bias control via dual-direction matching."
       }
     }
   },
   "claims": [
     {
       "claim": "Genetic-Instruct achieves 69.7% average accuracy across coding benchmarks, outperforming alternative synthetic generation methods and public datasets",
       "evidence": "Table 1: Genetic-Instruct 7.5M achieves 69.7% average vs best baseline Self-Instruct at 66.8% and best public dataset OpenCoder Stage 1 at 62.9%. Four benchmarks reported individually.",
       "supported": "moderate"
     },
     {
       "claim": "Scaling synthetic data improves model performance with diminishing returns beyond ~6M samples",
       "evidence": "Figure 2 shows coding accuracy rising from ~45% baseline to ~69% at 7.5M samples, with plateau visible after ~6M. Six generations of ~1.5M samples each.",
       "supported": "moderate"
     },
     {
       "claim": "Combining mutation and crossover operations yields better performance than either alone",
       "evidence": "Table 2: Combined approach achieves 68.0% average vs crossover-only 66.8% and mutation-only 66.6%, all at 4M samples with same base model.",
       "supported": "moderate"
     },
     {
       "claim": "Smaller generator models can produce competitive synthetic data",
       "evidence": "Table 3: Qwen-7B as generator yields 66.5% (Llama base) and 76.7% (Qwen base) vs Qwen-32B at 66.9% and 77.3% respectively.",
       "supported": "moderate"
     },
     {
       "claim": "Models trained on Genetic-Instruct data outperform Llama3.1-8B-Instruct",
       "evidence": "Table 1: Genetic-Instruct 7.5M achieves 69.7% average vs Llama 3.1 8B Instruct at 65.9%.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results are reported as point estimates without confidence intervals, standard deviations, or multiple runs. The margin between Genetic-Instruct (68.0%) and Self-Instruct (66.8%) at 4M samples is 1.2 percentage points — this could easily be within noise for single-run results."
     },
     {
       "flag": "Company evaluating own compute-intensive method",
       "detail": "All authors are NVIDIA employees. The proposed method requires massive GPU-scale parallelism (20 colonies, 7.5M LLM-generated samples). NVIDIA has commercial interest in demonstrating the value of compute-intensive approaches but does not acknowledge this conflict."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or discussion of scope boundaries. This is a significant omission for an empirical paper."
     },
     {
       "flag": "Baseline re-implementations by competing method authors",
       "detail": "The authors re-implement all baselines (WizardCoder, Self-Instruct, OSS-Instruct, INVERSE-INSTRUCT) without acknowledging the bias of implementing competing methods. INVERSE-INSTRUCT achieves only 41.1% in their re-implementation — an unusually poor result that warrants scrutiny."
     },
     {
       "flag": "Python-only evaluation with broad claims",
       "detail": "The title and abstract claim 'Coding Instructions for Large Language Models' generally, while all evaluation is on Python-only benchmarks (HumanEval/MBPP). The abstract does not bound claims to Python."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-instruct: Aligning language models with self-generated instructions",
       "authors": ["Yizhong Wang", "Yeganeh Kordi", "Swaroop Mishra"],
       "year": 2023,
       "relevance": "Foundational method for synthetic instruction generation using LLMs, serves as a baseline."
     },
     {
       "title": "WizardCoder: Empowering code large language models with evol-instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2024,
       "relevance": "Adapts Evol-Instruct for code generation via instruction mutation, key baseline in the synthetic coding data space."
     },
     {
       "title": "Magicoder: Empowering code generation with oss-instruct",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu"],
       "year": 2024,
       "relevance": "Generates coding instructions from open-source code snippets (OSS-Instruct), key baseline for code-inspired instruction generation."
     },
     {
       "title": "InverseCoder: Unleashing the power of instruction-tuned code LLMs with inverse-instruct",
       "authors": ["Yutong Wu", "Di Huang", "Wenxuan Shi"],
       "year": 2024,
       "arxiv_id": "2407.05700",
       "relevance": "Generates instructions from existing code, representing a code-to-instruction paradigm for synthetic data generation."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation, widely used for evaluating code LLMs."
     },
     {
       "title": "Is your code generated by chatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang"],
       "year": 2023,
       "relevance": "Introduces EvalPlus (HumanEval+, MBPP+) with additional test cases for more rigorous code evaluation."
     },
     {
       "title": "OpenCoder: The open cookbook for top-tier code large language models",
       "authors": ["Siming Huang", "Tianhao Cheng", "Jason Klein Liu"],
       "year": 2024,
       "relevance": "Open-source code LLM with public training data, serves as a baseline dataset comparison."
     },
     {
       "title": "Rethinking benchmark and contamination for language models with rephrased samples",
       "authors": ["Shuo Yang", "Wei-Lin Chiang", "Lianmin Zheng"],
       "year": 2023,
       "arxiv_id": "2311.04850",
       "relevance": "Proposes decontamination methodology adopted by this paper to prevent benchmark leakage in training data."
     },
     {
       "title": "The llama 3 herd of models",
       "authors": ["Aaron Grattafiori"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Base model (Llama 3.1 8B) used for fine-tuning experiments in the primary evaluation."
     },
     {
       "title": "WizardLM: Empowering large pre-trained language models to follow complex instructions",
       "authors": ["Can Xu", "Qingfeng Sun", "Kai Zheng"],
       "year": 2024,
       "relevance": "Introduces Evol-Instruct with meta-instructions for increasing instruction complexity, foundational to the mutation operation."
     }
   ]
 }