ai-research-survey

scan-v5.json (28417B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Learning Code Preference via Synthetic Evolution",
     "authors": [
       "Jiawei Liu",
       "Thanh Nguyen",
       "Mingyue Shang",
       "Hantian Ding",
       "Xiaopeng Li"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2410.03837",
     "doi": "10.48550/arXiv.2410.03837"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All major abstract claims are verified in the paper: 28.8% improvement (Table 3), 6-9x parameter match and 34x cost savings (Table 4), and 15.1-40.3% unsolved human tasks derived from accuracy scores in Table 3.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about CODEFAVOR improving preference accuracy, and Section 3.4 provides comprehensive ablation studies comparing individual data sources (Commit-Instruct vs Critic-Evol), modeling choices, and prompt variants across multiple base models.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Human performance findings (e.g., 'generalist programmers may struggle to accurately assess non-functional code properties') are stated broadly, but the study uses only 18 developers on Python tasks; Appendix A.6 acknowledges scope is limited to self-contained code snippets yet the abstract does not caveat this.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether CODEFAVOR improvements might stem from general fine-tuning domain adaptation rather than the specific synthetic data design; self-bias explanations appear only in narrow case studies, not as systematic alternative hypotheses.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes objective oracles (test execution, CPU instruction counts, static analysis) from human preference, and discusses where each proxy measure diverges from the others in Section 3.2-3.3.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.6 'Limitation and Future Work' contains three dedicated limitation points covering data scale, context dependency, and benchmark validity.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats named include modest training set size (62,236 samples), restriction to self-contained Python snippets, and subjectivity in human annotation for non-functional properties; these are more specific than boilerplate disclaimers.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "A.6 explicitly states CODEFAVOR focuses on self-contained code snippets and that real-world repository-level context is out of scope, and Table 1 bounds evaluation to four specific objectives.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure statement is present; the acknowledgment section thanks colleagues but does not disclose any funding source or grant.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated on the title page: UIUC for the first author and AWS AI Labs for the remaining four authors.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Four of five authors are from AWS AI Labs; the work directly relates to Amazon's commercial code assistant (CodeWhisperer is cited in the introduction), making the institutional funder non-independent of the outcome.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests declaration is present anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines 'code preference task' in Section 1 as choosing one code response over another given a criterion, and defines Commit-Instruct and Critic-Evol with precise input/output descriptions.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three numbered contributions are explicitly listed at the end of Section 1: the CODEFAVOR framework, the CODEPREFBENCH benchmark, and the empirical study of human/LLM preferences.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 'Related Work' situates CODEFAVOR relative to preference optimization (RLHF, DPO), LLM-as-judge approaches, and prior code preference work (Weyssow et al., McAleese et al.), explaining how this work differs and extends each.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly states 'We release the data and code at https://github.com/amazon-science/llm-code-preference' in the contributions section.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "CODEPREFBENCH is released at the same GitHub URL; seed datasets (EvalPlus, EvalPerf, CyberSecEval, BigCodeBench-Hard) are standard public benchmarks.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Training hardware (8xA100-40G), framework (Axolotl, DeepSpeed ZeRO-3), and vLLM versions are mentioned, but no requirements.txt, Dockerfile, or equivalent formal dependency specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Hyperparameters are listed in Table 8 and training setup described in A.2, but no step-by-step instructions for reproducing experiments are provided.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 reports range values (e.g., '84.9 (±9.4)', '59.7 (±37.0)') for stochastic models run multiple times, indicating variance across seeds.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests (t-tests, bootstrap intervals, etc.) are applied to the accuracy comparisons despite many comparative claims across models.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes are reported as relative percentage improvements (e.g., '9.3∼28.8% relatively', '34× cheaper') with baseline performance values provided for context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 1,364 benchmark tasks and 18-developer annotator pool are described but neither is justified by power analysis or sample size reasoning.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Variance (±ranges) is reported in Table 3 for models run with temperature > 0; entries without variance markers were run deterministically (greedy decoding).",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 compares against 10 open-weight and proprietary baselines including Claude 3.5 Sonnet, Gemini 1.5 Pro, Llama-3.1-405B, Mistral Large 2, and human annotators.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All baselines are state-of-the-art at time of writing (October 2024): Claude 3.5 Sonnet, Gemini 1.5, Llama 3.1 405B, Mistral Large 2, DeepSeek V2.5.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.4 provides extensive ablation: individual vs combined training data (Table 5), data mixture vs model merging, classification vs generation output, criteria variants (Table 6), draft/critic model pairing (Table 7).",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation uses four distinct metrics: correctness accuracy, efficiency accuracy, security accuracy, and human preference accuracy, plus cost-effectiveness (Table 4).",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "18 software developers annotated 145 human-preference pairs (Section 3.2) and served as a comparison baseline across all four benchmark categories.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "CODEPREFBENCH is the held-out evaluation set, explicitly separated from synthetic training data; Appendix A.5 measures and confirms low overlap between training and test sets.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "All results in Tables 3, 5, 6, 7 are broken down by correctness, efficiency, security, and human preference columns.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.4 provides extensive case studies of failure modes for LLMs (hallucination, missed edge cases), human annotators (overlooked details, tied on clear cases), and CODEFAVOR models across all three categories.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results include: code comments hurt accuracy (Table 6), using same draft/critic model degrades performance (Table 7), empty criteria substantially reduces security accuracy, and human annotators fail on non-functional objectives.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model versions are named throughout: Gemini 1.5 Pro 001, Gemini 1.5 Flash 001, Llama-3.1-405B-Instruct, Mistral Large 2 (123B), Codestral-22B-v0.1; vLLM versions also specified in A.3.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Complete prompt templates are provided: Listing 1 for evaluation, Figures 3-5 for Commit-Instruct and Critic-Evol data generation with full multi-turn examples.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 8 reports batch size (32), sequence length (2048), learning rate (5×10⁻⁶), scheduler (cosine annealing with 40 warm-up steps), and sequence packing settings.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; models are prompted directly or fine-tuned as classifiers/generators without multi-step orchestration.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Data processing steps are documented: positional bias augmentation (flipping code pair order), code comment clipping for Critic-Evol samples, filtering criteria for both Commit-Instruct (8.1% filtered) and Critic-Evol (17.9% filtered).",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "CODEPREFBENCH data and training datasets are released at the GitHub URL, enabling independent access to the raw preference pairs.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 details benchmark construction for each category: source datasets, pair selection method, oracle label generation, tie exclusion criteria, and positional bias shuffling.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": true,
           "answer": false,
           "justification": "The 18 developers' credentials are described (CS degrees, experience, proficiency) but recruitment method (how they were sourced, whether compensated, selection criteria) is not explained.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Both synthetic data pipelines (Commit-Instruct and Critic-Evol) are documented step-by-step in Section 2.2 with full prompt implementations in Figures 3-5.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training data cutoffs for the evaluated LLMs (Claude 3.5 Sonnet, Llama 3.1, etc.) are never stated, despite these models being evaluated on benchmarks (EvalPlus, LBPP) that may have appeared in their pretraining.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.5 'Quantifying Contamination' uses Levenshtein similarity analysis to measure overlap between synthetic training data and the evaluation benchmark, finding 0.1-1.7% above 80 similarity.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "The contamination analysis in A.5 explicitly measures training-test code similarity and references Riddell et al. (2024) as a framework, confirming the evaluation set is largely contamination-free from training data.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": true,
           "answer": false,
           "justification": "No pre-registration is mentioned for the human annotation study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": true,
           "answer": false,
           "justification": "No IRB approval or ethics review is mentioned despite involving 18 human participants in a formal annotation study.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 reports education (2/3 with CS degrees), experience (95% with 2+ years), and Python proficiency levels (43% advanced, rest middle-level).",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": true,
           "answer": false,
           "justification": "No formal inclusion or exclusion criteria for the 18-developer annotation team are stated.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": true,
           "answer": false,
           "justification": "Task assignment to annotators is not described as randomized; no randomization procedure is mentioned.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": true,
           "answer": false,
           "justification": "No blinding of annotators to model identities or task order is mentioned.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": true,
           "answer": false,
           "justification": "No attrition reporting; 145 final preference pairs are described as filtered for non-conflicting preferences, but annotator dropout is not discussed.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 provides normalized per-sample costs comparing human annotation ($6.1), Llama-405B (1200x CODEFAVOR cost), and CODEFAVOR models (baseline 1x).",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training hardware (8xA100-40G) is mentioned but total GPU-hours, wall-clock time, or dollar cost for model training is not reported.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "CODEFAVOR improves model-based code preference accuracy by up to 28.8% relatively over base models",
       "evidence": "Table 3 shows Mistral-7B improving from 64.5% to 74.3% average accuracy (+15.2pp); Gemma-2-9B improving from 60.1% to 77.4% (+28.8% relative stated in abstract/text)",
       "supported": "strong"
     },
     {
       "claim": "CODEFAVOR models match the performance of models 6-9x larger while being 34x more cost-effective",
       "evidence": "Table 3 shows 12B CODEFAVOR models achieving 77-77.7% average, near Llama-3-70B-Instruct (76.1%); Table 4 shows Mistral Nemo CODEFAVOR is 34x cheaper than Llama-70B",
       "supported": "strong"
     },
     {
       "claim": "Human preference is sub-optimal for non-functional code objectives (efficiency and security)",
       "evidence": "Table 3: human accuracy on efficiency (74.9%) is surpassed by Mistral Large 2 (81.2%); human security accuracy (59.7%) is the lowest of all approaches due to 73.9% of pairs being marked as tied",
       "supported": "strong"
     },
     {
       "claim": "15.1-40.3% of code preference tasks remain unsolved by developer agreement despite 23.4 person-minutes per task",
       "evidence": "Derived from Table 3: correctness 84.9% solved (15.1% unsolved), security 59.7% solved (40.3% unsolved); annotation time of 7.8 min/developer × 3 annotators = 23.4 minutes",
       "supported": "strong"
     },
     {
       "claim": "Code comments negatively impact preference model accuracy, particularly for correctness tasks",
       "evidence": "Table 6: evaluating default models (trained without comments) with comments causes 6.2-10.4% drop in correctness accuracy; training and evaluating with comments causes 6-7% drop in overall accuracy",
       "supported": "strong"
     },
     {
       "claim": "Using different draft and critic models (8B draft, 70B critic) outperforms using the same model for both roles in Critic-Evol",
       "evidence": "Table 7: 8B/70B pairing achieves 75.3% average vs 68.2% for 8B/8B and 73.4% for 70B/70B; attributed to LLM self-bias in same-model conditions",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "CODEFAVOR, trained on synthetic code evolution data from commits and LLM critiques, enables 7-12B models to match 70B+ model preference accuracy while being 34x cheaper, achieving up to 28.8% relative improvement over base models. Human annotators excel at code correctness (84.9% accuracy) but are surprisingly weak on non-functional objectives: security accuracy is 59.7% (lower than all LLMs) because 73.9% of security pairs are marked tied, and efficiency accuracy (74.9%) is surpassed by multiple models. Model merging of two complementary training datasets outperforms data mixture for classification models, and code comments systematically hurt preference accuracy by allowing LLMs' self-bias to prefer well-commented but incorrect code.",
   "red_flags": [
     {
       "flag": "Institutional conflict of interest",
       "detail": "Four of five authors are from AWS AI Labs; the paper advances technology directly applicable to Amazon's CodeWhisperer product, yet no competing interests statement is provided."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative accuracy claims across models are made without statistical significance tests; many differences (e.g., 1-2pp) may not be meaningful given the benchmark sizes."
     },
     {
       "flag": "Human study lacks ethics disclosure",
       "detail": "18 developers participated in a formal annotation study with tracked timing and annotation behavior, but no IRB approval, consent procedures, or compensation information is disclosed."
     },
     {
       "flag": "No inference cost for model training",
       "detail": "Training compute budget (GPU-hours, total cost) is not disclosed, making the 'cost-effective' framing incomplete — CODEFAVOR requires upfront training that is not accounted for in Table 4."
     },
     {
       "flag": "Generalization limited to Python self-contained code",
       "detail": "All results are on Python snippets; claims about human limitations and LLM preference behavior are stated broadly but derive from a narrow, self-contained snippet evaluation context."
     }
   ],
   "cited_papers": [
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation (EvalPlus)",
       "relevance": "Primary evaluation benchmark for code correctness preference tasks in CODEPREFBENCH"
     },
     {
       "title": "Evaluating Language Models for Efficient Code Generation (EvalPerf)",
       "relevance": "Source of the code efficiency preference subset with CPU instruction count oracles"
     },
     {
       "title": "Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models",
       "relevance": "Source of security-vulnerable code pairs for CODEPREFBENCH's security category"
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "relevance": "Establishes the LLM-as-judge and pairwise preference evaluation framework this paper extends to code"
     },
     {
       "title": "CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences",
       "relevance": "Closely related prior work on LLM-as-judge for code quality; CODEFAVOR addresses its limitation of not training preference models from scratch"
     },
     {
       "title": "LLM Critics Help Catch LLM Bugs (CritiGPT)",
       "relevance": "Related work on training critic models to catch code bugs; CODEFAVOR confirms and extends their findings on human preference limitations"
     },
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "relevance": "Foundational method (DPO) for preference optimization that the code preference data from this paper could feed"
     },
     {
       "title": "Prometheus: Inducing Fine-Grained Evaluation Capability in Language Models",
       "relevance": "Prior work on training LLM judges that the CODEFAVOR pairwise modeling builds upon"
     },
     {
       "title": "Quantifying Contamination in Evaluating Code Generation Capabilities of Language Models",
       "relevance": "Provides the contamination quantification methodology (Levenshtein similarity) used in Appendix A.5"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly applicable to code review automation, RLHF data collection for code LLMs, and replacing expensive human annotation — released code and benchmark enable immediate adoption."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that human developers are worse than LLMs at evaluating code security (59.7% vs near-saturation for most models) and that code comments hurt LLM preference accuracy are counterintuitive results."
     },
     "fear_safety": {
       "score": 1,
       "justification": "The code security evaluation component touches on vulnerability detection but does not raise broader AI risk concerns; security is framed as a benchmark category, not a safety issue."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Comparison between proprietary models (Claude 3.5 Sonnet, Gemini) and open models provides some competitive interest, but no major controversy angle."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and benchmark are released at GitHub, allowing practitioners to run CODEPREFBENCH evaluations and fine-tune models using the released synthetic datasets."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "AWS AI Labs affiliation and evaluation of Claude 3.5 Sonnet, Gemini 1.5, and Llama 3.1 405B lend brand recognition; not from top-tier academic lab but from major industry player."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "42898158",
         "title": "Chrono: A Peer-to-Peer Network with Verifiable Causality",
         "points": 8,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=42898158"
       },
       {
         "hn_id": "37882946",
         "title": "Next-Generation OS Physical Memory Management for Terabyte-Scale NVMMs",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37882946"
       },
       {
         "hn_id": "33314496",
         "title": "A study of malicious CVE proof of concept exploits in GitHub",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=33314496"
       },
       {
         "hn_id": "41746915",
         "title": "Code-Survey: An LLM-Driven Methodology for Analyzing Large-Scale Codebases",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=41746915"
       },
       {
         "hn_id": "42683996",
         "title": "Cost-Performance Evaluation of General Compute: AWS, Azure, GCP, and OCI",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42683996"
       },
       {
         "hn_id": "42076089",
         "title": "Detection of Thermal Emission at mm Wavelengths from Low-Earth Orbit Satellites",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42076089"
       },
       {
         "hn_id": "30476106",
         "title": "Application-Oriented Performance Benchmarks for Quantum Computing",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=30476106"
       }
     ],
     "top_points": 8,
     "total_points": 18,
     "total_comments": 2
   }
 }