ai-research-survey

scan.json (29296B)

---

 {
   "paper": {
     "title": "Exploring Data-Efficient Adaptation of Large Language Models for Code Generation",
     "authors": [
       "Xue Jiang",
       "Yihong Dong",
       "Zhiyuan Fan",
       "Zhi Jin",
       "Wenpin Jiao",
       "Ge Li"
     ],
     "year": 2025,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2403.00046",
     "doi": "10.1145/3772721"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DEED, an error-driven learning approach for data-efficient LLM adaptation, achieves an average 46.2% relative improvement in Pass@1 over fine-tuning baselines across five Python code generation benchmarks with limited training data. Self-revision by the same model produces better training data than revision by more powerful external LLMs like ChatGPT, suggesting data suitability matters more than quantity. The method generalizes across four LLMs (CodeGen-2B/6B, Llama-7B, CodeLlama-7B) and shows iterative adaptation stabilizes after two rounds.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All five evaluation datasets (HumanEval, MBPP, HumanEval-ET, MBPP-ET, DataScience) are publicly available benchmarks that were not modified by the authors."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a single A6000 GPU' (Section 4.2) but provides no requirements.txt, library versions, or environment setup details beyond the hardware."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts to replicate experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables report point estimates only (e.g., '38.6%'). Despite averaging over five test runs (Section 7), no confidence intervals, error bars, or ± notation are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DEED 'performs significantly better' (Section 5.1) and reports 'significant relative improvements' but uses no statistical tests (no p-values, t-tests, or bootstrap tests) to support these claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative improvements are reported with baseline context throughout (e.g., 'relative improvements of 29.5%, 33.0%, 27.1%, 37.6%, and 103.8%, respectively, when compared to the best-performing baseline' in Section 5.1). Tables provide both baseline and DEED numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why min(200, 40%*D) was chosen as the training split size, or why 5 datasets were selected. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 states 'each experiment is run five times, and its average result is reported' but no standard deviation, variance, or spread measure is reported across these runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six baselines are compared: Direct Generation, Fine-tuning (Full), Fine-tuning (LoRA), Few-shot Prompting, Self-Refine, and Self-Debug (Section 5.1, Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Self-Refine (Madaan et al., 2023) and Self-Debug (Chen et al., 2023) are contemporary methods. Fine-tuning and LoRA are standard and still widely used."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ6 (Section 5.6, Table 6) performs ablation on Self-Revise input components (correct solution, error messages, failed test cases). RQ3 (Table 3) compares training data variants. RQ4 (Table 4) studies iteration count."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1, Pass@5, and Pass@10 are reported across all experiments (Tables 1-6). Pass@any is additionally used for automatic code revision evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via test case execution (Pass@k). While manual inspection of Self-Revise outputs appears in Figure 4, this examines the revision process rather than systematically evaluating the final model's output quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'We sample min(200, 40%*D) problems from the datasets as D_train, while the remaining problems serve as D_test.' Explicit train/test separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (5 benchmarks in Table 1), by model (4 LLMs in Table 2), by training data variant (Table 3), by iteration count (Table 4), and by revision model (Table 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 discusses why ChatGPT-based revision underperforms self-revision (tendency to make large changes, capability gap). Section 5.1 notes Self-refine and Self-debug 'underperform on small LLMs.' Figure 4 shows Self-Revise (FSP) copying code rather than minimally revising."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: Fine-tuning (LoRA) is less effective than Fine-tuning (Full) (Section 5.1); DEED ∪ D_train is not as effective as DEED alone showing some samples have 'negative effects' (Section 5.3); ChatGPT revision doesn't improve the final model as expected (Section 5.5); Pass@10 oscillates in iterations 2-4 (Table 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'average relative improvement of 46.2% in Pass@1.' The five relative improvements from Table 1 (29.5%, 33.0%, 27.1%, 37.6%, 103.8%) average to exactly 46.2%. Other claims about Self-Revise effectiveness and cross-LLM applicability are supported by Tables 3 and 2 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('DEED improves performance') are supported by controlled ablation studies (RQ6, Table 6) that isolate individual component contributions through single-variable removal. The training variant study (RQ3, Table 3) controls for data differences. Ablation design is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Code Generation' broadly, but all five benchmarks are Python-only (HumanEval, MBPP, MBPP-ET, HumanEval-ET, DataScience). The paper never acknowledges this language limitation. The Limitations section (Section 8) bounds to 'low-resource scenarios' and 'requires test cases' but does not bound by programming language."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses threats to validity including hyperparameter sensitivity and dataset generalizability. Section 5.5 considers two alternative explanations for ChatGPT revision underperformance: (1) tendency to disregard minimal revision instructions, and (2) capability gap between models. Section 2 provides a representation-space analysis as mechanistic explanation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@k (test case passing rate) and claims improvement in 'code generation performance.' The claims match the measurement granularity — they do not overclaim 'code quality' or 'developer productivity' from Pass@k numbers."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models are identified by name and size (CodeGen-2B/6B, Llama-7B, CodeLlama-7B) with references, but 'ChatGPT' and 'GPT-3.5-turbo' used in RQ5 lack specific API versions or snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C provides the instruction text for code revision, and Figure 3 shows the template structure with placeholders {r_i}, {g_i}, {c'_i}, {m_i}, {t_i}. However, the few-shot prompting baseline prompt is not provided, and Self-Revise (FSP) examples used in the prompt are not fully listed."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports comprehensive hyperparameters: AdamW optimizer with β1=0.9, β2=0.9, learning rate 5e-6 (full) / 2e-4 (LoRA), batch size 1, gradient accumulation 32, 10 epochs, LoRA rank 128, α=8, temperature 0.8, sampling counts (5 for error collection, 30 for revision), 2 iterations, max generation length 1024."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. DEED is a fine-tuning pipeline, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the data split: 'We sample min(200, 40%*D) problems from the datasets as D_train.' Section 3.1-3.2 details the error code collection and revision pipeline including sampling strategy, selection criteria (highest generation probability for errors, minimum Levenshtein distance for revisions), and filtering (test-case-based acceptance/rejection)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Threats to Validity) provides substantive discussion across three categories (external, internal, construct validity). Section 8 (Limitations) discusses two specific limitations of the approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses threats specific to this study: use of five specific public benchmarks for generalizability, sensitivity to hyperparameters with only small-range grid search performed, and reliance on Pass@k metric with its unbiased estimator."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states two specific scope boundaries: (1) 'our approach requires test cases' during training, and (2) 'DEED is only used in low-resource scenarios.' However, the Python-only limitation is not explicitly stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (generated error codes, revisions, model outputs, fine-tuned model weights) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes error code collection via rejection sampling with test evaluation. Section 3.2 describes the revision process via acceptance sampling. Section 4.1 describes dataset sourcing from public benchmarks with specific references."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public code generation benchmarks (HumanEval, MBPP, DataScience, EvoCodeBench)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: dataset splitting (Section 4.1), error code collection with selection criteria (Section 3.1), automatic revision with acceptance sampling and filtering (Section 3.2), model optimization (Section 3.3), and iterative adaptation with experience replay (Section 3.4). Algorithm 1 summarizes the complete process."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists: National Key R&D Program (Grant No. 2023YFB4503801), National Natural Science Foundation of China (Grant No. 62192733, 62192730, 62192731), and Major Program of Hubei Province (No. 2023BAA024)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Key Laboratory of High Confidence Software Technologies (Peking University), Ministry of Education; School of Computer Science, Peking University. No product of theirs is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Chinese government research agencies (NSFC, National Key R&D Program, Hubei Province) which have no financial stake in whether DEED outperforms baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for any of the models used (CodeGen-2B/6B, Llama-7B, CodeLlama-7B). Cannot assess whether benchmark data was in pre-training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval, MBPP, or other benchmark problems appeared in the pre-training data of the models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) were published before the training of CodeGen (2022-2023) and other models used. No contamination analysis is performed despite this clear temporal risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes DEED incurs 'no additional resource or time costs' at inference compared to direct generation (Section 6.2), but no actual inference latency, cost per example, or token counts are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 states 'a single A6000 GPU' was used, but total GPU hours, training time, or compute budget are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 states experiments are 'run five times' and averaged, but no results across individual runs or seed sensitivity analysis is reported. The five runs appear to be for the unbiased Pass@k estimator sampling, not separate training runs with different seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states: 'each experiment is run five times, and its average result is reported.' Section 4.3 also clarifies n=50 samples generated per problem for Pass@k calculation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 states 'we only do a small-range grid search on hyperparameters, including iterations of DEED, learning rates, and training epochs' but the number of configurations tried and total compute spent on search are not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The iteration count is justified via RQ4 (Table 4, selecting 2 iterations based on Pass@10 oscillation). However, for other hyperparameters (learning rate, epochs, LoRA rank), no selection methodology is described — it's unclear whether selection used validation or test performance."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 5 datasets, 4 models, 6 baselines, and multiple ablation variants without any statistical tests, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines themselves and compare against their own DEED method. No acknowledgment of author-evaluation bias or independent evaluation is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute budget. DEED involves iterative fine-tuning, error collection, and revision sampling which likely uses substantially more compute than single-pass fine-tuning, but this is not quantified or compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval, MBPP, or DataScience actually measure 'code generation capability in specific scenarios.' The paper uses these benchmarks to simulate data-scarce scenarios without questioning whether synthetic scarcity reflects real-world constraints."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. DEED is a fine-tuning method, not a scaffolded agent system."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) were published before the models' training periods. No discussion of whether model pre-training data included benchmark solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Self-Revise uses correct solutions and test cases from the training split during the revision process. While this is the method design, no discussion of whether the evaluation setup (using the same benchmark's held-out problems) leaks information through structural similarity."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper splits datasets into train/test by random sampling but does not discuss whether problems in the same benchmark share structural similarities, common patterns, or come from the same source distributions that could create non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used (no canary strings, membership inference, n-gram overlap analysis, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "DEED achieves an average relative improvement of 46.2% in Pass@1 compared to the best-performing baseline across five code generation benchmarks.",
       "evidence": "Table 1 shows relative improvements of 29.5% (HumanEval), 33.0% (HumanEval-ET), 27.1% (MBPP), 37.6% (MBPP-ET), and 103.8% (DataScience) over Fine-tuning (Full), averaging 46.2%.",
       "supported": "strong"
     },
     {
       "claim": "Self-Revise produces revised code that is more efficient for model optimization than code samples from datasets.",
       "evidence": "Table 3 (Section 5.3) shows DEED (32.8% Pass@1) outperforms Raw D_train (25.8%) despite using less training data. Also, DEED ∪ D_train (29.2%) underperforms DEED alone, suggesting some dataset samples have negative training effects.",
       "supported": "moderate"
     },
     {
       "claim": "DEED consistently enhances performance across different LLMs of varying sizes and architectures.",
       "evidence": "Table 2 (Section 5.2) shows improvements on CodeGen-2B (+27.1%), CodeGen-6B (+25.6%), Llama-7B (+32.5%), and CodeLlama-7B (+25.2%) on MBPP dataset.",
       "supported": "moderate"
     },
     {
       "claim": "Using the same LLM for revision (Self-Revise) yields better final model performance than using more powerful external LLMs like ChatGPT.",
       "evidence": "Table 5 (Section 5.5) shows Self-Revise (FT) achieves 32.8% Pass@1 on the final model vs. 27.0% for ChatGPT-based revision and 29.0% for GPT-3.5-turbo, despite ChatGPT achieving far higher revision accuracy (61.4% vs 3.9% Pass@1 on MRevise).",
       "supported": "moderate"
     },
     {
       "claim": "Revised codes are closer to error codes than dataset samples in the model's representation space, explaining why error-driven learning is more efficient.",
       "evidence": "Section 2 preliminary study shows average Euclidean distance between error codes and revisions is 6.39, vs. 12.35 between error codes and dataset samples, using CodeGen-2B on MBPP.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification despite multiple runs",
       "detail": "The paper runs each experiment five times but reports only averaged point estimates with no standard deviation, confidence intervals, or error bars. Claims of 'significant improvement' are made without any statistical tests."
     },
     {
       "flag": "Most ablations conducted on single dataset",
       "detail": "RQ2 (different LLMs), RQ3 (training variants), RQ4 (iterations), RQ5 (revision models), and RQ6 (ablation) are all conducted exclusively on MBPP. Only RQ1 uses all five benchmarks. Generalizability of component-level findings is unverified."
     },
     {
       "flag": "No contamination analysis",
       "detail": "HumanEval and MBPP (both 2021) were published before CodeGen's training. The paper does not analyze whether benchmark solutions appeared in pre-training data, which could inflate absolute performance numbers for all methods."
     },
     {
       "flag": "Simulated data scarcity may not reflect real scenarios",
       "detail": "The paper claims to address 'specific scenarios' like aerospace and medical devices (Section 1) but evaluates on artificially subsampled general Python benchmarks. Whether findings transfer to genuine domain-specific code generation with inherently scarce data is untested."
     },
     {
       "flag": "No code or model release",
       "detail": "Despite proposing a novel method, no source code, trained models, or experimental artifacts are released, preventing independent verification of the results."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and Codex; foundational for LLM code generation evaluation methodology."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi"],
       "year": 2023,
       "relevance": "Open-source code LLM used as the primary base model in all experiments."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open code LLM used as one of the evaluated models; key baseline for code generation capability."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["Edward J. Hu", "Yelong Shen", "Phillip Wallis"],
       "year": 2022,
       "relevance": "Parameter-efficient fine-tuning method used as both a baseline and an adaptation technique within DEED."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Prompting-based iterative code refinement baseline; demonstrates LLM self-improvement capabilities."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "LLM self-debugging using execution feedback; directly compared as a baseline for code correction approaches."
     },
     {
       "title": "Self-Edit: Fault-Aware Code Editor for Code Generation",
       "authors": ["Kechi Zhang", "Zhuo Li", "Jia Li"],
       "year": 2023,
       "relevance": "Trains a separate editor for code correction; related approach using error-aware training for code generation."
     },
     {
       "title": "CYCLE: Learning to Self-Refine the Code Generation",
       "authors": ["Yangruibo Ding", "Marcus J. Min", "Gail E. Kaiser"],
       "year": 2024,
       "relevance": "Concurrent work enhancing LLM self-refinement capability for code correction using test results."
     },
     {
       "title": "EvoCodeBench: An Evolving Code Generation Benchmark with Domain-Specific Evaluations",
       "authors": ["Jia Li", "Ge Li", "Xuanming Zhang"],
       "year": 2024,
       "relevance": "Project-level code generation benchmark addressing data leakage; used for supplementary evaluation in Appendix B."
     },
     {
       "title": "WizardCoder: Empowering Code Large Language Models with Evol-Instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Data augmentation approach for code LLMs via evolved instructions; contrasting approach to DEED's error-driven learning."
     },
     {
       "title": "Magicoder: Empowering Code Generation with OSS-Instruct",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu"],
       "year": 2024,
       "relevance": "Instruction synthesis from open-source code for LLM training; alternative data-efficient approach for code generation."
     },
     {
       "title": "Improving Code Generation by Training with Natural Language Feedback",
       "authors": ["Angelica Chen", "Jérémy Scheurer", "Tomasz Korbak"],
       "year": 2023,
       "arxiv_id": "2303.16749",
       "relevance": "ILF method using human feedback to refine code generation; related approach requiring more human involvement than DEED."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Landmark LLM code generation system demonstrating competition-level performance; established sampling and filtering methodology."
     }
   ]
 }