ai-research-survey

scan.json (21438B)

---

 {
   "paper": {
     "title": "Competition-Level Code Generation with AlphaCode",
     "authors": ["Yujia Li", "David Choi", "Junyoung Chung", "Nate Kushman", "Julian Schrittwieser", "Rémi Leblond", "Tom Eccles", "James Keeling", "Felix Gimeno", "Agustin Dal Lago", "Thomas Hubert", "Peter Choy", "Cyprien de Masson d'Autume", "Igor Babuschkin", "Xinyun Chen", "Po-Sen Huang", "Johannes Welbl", "Sven Gowal", "Alexey Cherepanov", "James Molloy", "Daniel J. Mankowitz", "Esme Sutherland Robson", "Pushmeet Kohli", "Nando de Freitas", "Koray Kavukcuoglu", "Oriol Vinyals"],
     "year": 2022,
     "venue": "arXiv / Science",
     "arxiv_id": "2203.07814"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The CodeContests dataset is released on GitHub (https://github.com/deepmind/code_contests), referenced in Section 3.2. However, the AlphaCode model code itself is not released. The dataset release counts as a partial artifact release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The CodeContests dataset is publicly released on GitHub (Section 3.2, footnote 1). The pre-training GitHub dataset is described but based on a public snapshot."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions JAX, Haiku, TPUv4, bfloat16 precision (Section 4.1), but no requirements.txt, Dockerfile, or detailed environment specification is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The model weights are not released, and no README with commands to reproduce results is mentioned."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports 95% confidence intervals for ablation results, e.g., '6.7% (6.5-6.8)'. Bootstrapping is used for metric estimation (Section 2.2, Appendix A.3)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal statistical significance tests (p-values, t-tests, etc.) are reported. Comparisons rely on confidence intervals and point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout, e.g., Table 8 shows ablation improvements from 15.2% to 24.1% (10@100k), and Table 5 shows solve rates across conditions."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The validation set has 117 problems and test set 165 problems. No justification is given for why these sizes are sufficient, nor is a power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports confidence intervals from multiple fine-tuning runs ('at least 3 different models from the same pre-trained checkpoint'). Codeforces evaluation was repeated 3 times (Section 5.1)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons against GPT-Neo 2.7B and Codex 12B on APPS (Table 10), and against decoder-only and standard multi-head attention architectures (Table 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Codex (Chen et al., 2021) and GPT-Neo were the state-of-the-art code generation models at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation study in Section 5.3 and Table 8, showing the contribution of MLM, tempering, tags/ratings, value conditioning, GOLD, and clustering."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: pass@k, 10@k, n@k at various sample budgets. Also reports false positive rates (Table 2) and per-category breakdowns."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports manual examination of 50 solutions per dataset to check false positive rates. Section 6.1 includes qualitative analysis of 50 model-generated solutions for copying behavior."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Strict temporal split: training data before 2021/07/14, validation 2021/07/15-2021/09/20, test after 2021/09/21 (Section 3.2). Results reported on both validation and test sets (Table 5)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 11 shows solve rates across 10 problem types (greedy, math, DP, etc.) at different model sizes. APPS results broken down by difficulty level (Table 10). Appendix E.2 has difficulty rating buckets."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 extensively discusses capabilities and limitations. Section 6.3 analyzes sensitivity to description changes. Table 11 shows weak areas (DP, constructive algorithms). Dead code analysis in Section 6.2."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 shows validation loss is a poor proxy for solve rate. The 41B model was undertrained due to resource limitations (Section 4.2). Top-k and nucleus sampling did not improve performance (Section 4.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of top 54.3% ranking are supported by Table 4. The three key components (dataset, architecture, sampling+filtering) are validated through ablations in Section 5.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by controlled ablation studies (Table 8) where components are added one at a time. Architecture comparisons use matched compute (Table 6)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is specific about the evaluation setting: Codeforces competitions, CodeContests dataset, APPS benchmark. Section 8.1 explicitly notes 'there are few direct applications of this work outside of competitive programming.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 investigates whether the model simply copies from training data. Section 6.3 tests whether the model exploits problem structure shortcuts. Section 6.5 discusses the disconnect between loss and solve rate."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Custom models with exact architecture specifications in Table 3: AlphaCode 300M, 1B, 3B, 9B, 41B with detailed parameter counts, hidden dimensions, head counts, training steps, and tokens."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows the metadata format. Appendix F provides complete examples of model prompts and samples. The exact input format is documented."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Extensive hyperparameter reporting: learning rates, β values, weight decay, temperature (T=0.2 training, T'=0.25 sampling), batch sizes, training steps (Sections 4.2, 4.3, 4.4, Table 3)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is described in detail: pre-training → fine-tuning → large-scale sampling → filtering on example tests → clustering via generated test inputs → submission selection (Section 4, Figure 4)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pre-training data filtering documented (Section 3.1): files >1MB removed, lines >1000 chars removed, duplicates removed. CodeContests cleaning in Appendix B.2. Generated test case procedure in Section 3.2.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'AlphaCode's capabilities & limitations' and discusses limitations extensively. Section 8 'Broader impact' discusses risks. Section 6.5 discusses loss as a poor proxy."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: false positive rates in evaluation (Table 2, Section 3.2.1), temporal split limitations, self-selected participant pool for Codeforces comparison (Section 5.1), 41B model being undertrained."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.1 explicitly states 'there are few direct applications of this work outside of competitive programming.' The evaluation is bounded to Codeforces competitions with >5000 participants. Self-selection of competitors is noted."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The CodeContests dataset is publicly released on GitHub (Section 3.2, footnote 1), including problems, solutions, and test cases."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: GitHub snapshot date (2021/07/14), Codeforces scraping, integration with Description2Code and CodeNet datasets, temporal split dates."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited. Data comes from public GitHub repositories and Codeforces platform. The Codeforces evaluation uses existing competition infrastructure."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: GitHub filtering (Section 3.1), CodeContests curation with cleaning (Appendix B.2), generated test cases (Section 3.2.1), problem filtering criteria (≥5 hidden tests, ≥2 different outputs)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper is from DeepMind (Google). Affiliation is clearly stated: '© 2022 DeepMind. All rights reserved.' All authors are DeepMind employees/interns."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors affiliated with DeepMind. Footnotes note Igor Babuschkin moved to OpenAI, Xinyun Chen was a DeepMind intern from UC Berkeley."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "DeepMind/Google funds the research and has a commercial interest in demonstrating AI capabilities. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training data cutoff explicitly stated: GitHub snapshot on 2021/07/14, all CodeContests training data publicly released on or before 2021/07/14 (Section 3.2)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes temporal split to prevent leakage. Section 6.1 analyzes copying from training data with longest common substring analysis. Appendix B.3 provides further analysis."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal split ensures all evaluation problems post-date training data (Section 3.2). Section 6.1 provides extensive analysis showing the model does not copy solutions. Problems are newly created for each competition."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The paper evaluates an AI system on programming competitions."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The system is evaluated against existing competition data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving humans."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving humans."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports sampling speed in TPU seconds per sample. Figure 7(b) plots solve rate vs. sampling TPU-seconds per problem. Section 8.2 notes 'hundreds of petaFLOPS days' for sampling and training."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training compute shown in Figure 7(a) in TPU-days. Table 3 lists training steps and tokens. Section 8.2 states 'Both sampling and training from our model required hundreds of petaFLOPS days.' TPUv4 accelerators mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "AlphaCode achieved an average ranking within the top 54.3% in simulated Codeforces competitions with more than 5,000 participants.",
       "evidence": "Table 4 shows estimated percent rankings across 10 contests. Average estimated ranking is 54.3%. Three evaluation runs were conducted to measure variance (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "AlphaCode achieved an estimated Codeforces rating of 1238, within the top 28% of users who participated in a contest in the last 6 months.",
       "evidence": "Section 5.1 and Figure 1(b) show the rating estimate. The 28% figure refers to active users, a self-selected subset (footnote 4).",
       "supported": "strong"
     },
     {
       "claim": "Three key components are critical for performance: clean dataset, efficient transformer architectures, and large-scale sampling with filtering.",
       "evidence": "Table 8 build-up ablation shows each component contributes (10@100k from 15.2% to 24.1%). Table 7 shows pre-training dataset impact. Figure 8 shows filtering/clustering importance.",
       "supported": "strong"
     },
     {
       "claim": "AlphaCode does not simply copy solutions from the training data.",
       "evidence": "Section 6.1 analyzes longest common substrings between model solutions and training data, finding similar distributions to human solutions. Figure 9 and qualitative analysis of 50 solutions in Figure 10.",
       "supported": "strong"
     },
     {
       "claim": "CodeContests reduces false positive rates from 30-60% in existing datasets to 4%.",
       "evidence": "Table 2 compares false positive rates: APPS 60%, HumanEval 30%, CodeContests raw 62%, CodeContests 4%. Based on manual examination of 50 problems per dataset.",
       "supported": "strong"
     },
     {
       "claim": "Solve rates scale log-linearly with the number of samples.",
       "evidence": "Figure 6 shows approximately log-linear scaling for both 10@k and pass@k metrics across model sizes. However, 10@k curves bend slightly at high sample budgets.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "AlphaCode is the first AI system to achieve competitive-level performance in programming competitions, ranking in the top 54.3% of Codeforces participants. The system relies on massive sampling (up to 1M samples per problem) followed by filtering and clustering to select 10 submissions, achieving 34.2% solve rate on the CodeContests validation set. Key innovations include a clean competitive programming dataset with temporal splits and generated test cases reducing false positives to 4%, and sampling-efficient encoder-decoder architectures with multi-query attention. Ablation studies show that each component (MLM, tempering, value conditioning, GOLD, clustering) contributes meaningfully, and solve rates scale log-linearly with both model parameters and sample budget.",
   "red_flags": [
     {
       "flag": "Company evaluating own system",
       "detail": "All authors are DeepMind employees. While the evaluation methodology is rigorous (using Codeforces platform and temporal splits), there is an inherent conflict of interest in DeepMind evaluating its own system's capabilities."
     },
     {
       "flag": "Self-selected comparison population",
       "detail": "The 'top 28% of users' claim is among users who participated in contests in the last 6 months, which is a self-selected group of active competitive programmers. The paper acknowledges this but the framing could overstate the achievement."
     },
     {
       "flag": "Massive compute requirements not fully quantified",
       "detail": "The paper mentions 'hundreds of petaFLOPS days' for both training and sampling but does not provide a precise total compute budget or cost estimate, making practical reproducibility assessment difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark, a key baseline and evaluation framework for code generation."
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "authors": ["Dan Hendrycks"],
       "year": 2021,
       "relevance": "Introduces the APPS dataset of programming problems used as a comparison benchmark in this paper."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Early work on using large language models for program synthesis, establishing baseline capabilities."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom B. Brown"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "GPT-3 paper establishing the foundation for large language model capabilities including code generation."
     },
     {
       "title": "Training Verifiers to Solve Math Word Problems",
       "authors": ["Karl Cobbe"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Demonstrates majority voting and verifier-based sample selection, related to AlphaCode's filtering approach."
     },
     {
       "title": "Extracting Training Data from Large Language Models",
       "authors": ["Nicholas Carlini"],
       "year": 2021,
       "relevance": "Addresses memorization and data extraction concerns in large language models, relevant to AlphaCode's copying analysis."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng"],
       "year": 2020,
       "relevance": "Pre-trained model for code understanding, part of the foundation for transformer-based code generation."
     },
     {
       "title": "Project CodeNet: A Large-Scale AI for Code Dataset for Learning a Diversity of Coding Tasks",
       "authors": ["Ruchir Puri"],
       "year": 2021,
       "relevance": "Large-scale code dataset used as part of AlphaCode's CodeContests training set."
     }
   ]
 }