ai-research-survey

scan.json (36044B)

---

 {
   "paper": {
     "title": "Evaluating Large Language Models Trained on Code",
     "authors": [
       "Mark Chen",
       "Jerry Tworek",
       "Heewoo Jun",
       "Qiming Yuan",
       "Henrique Ponde de Oliveira Pinto",
       "Jared Kaplan",
       "Harri Edwards",
       "Yuri Burda",
       "Nicholas Joseph",
       "Greg Brockman",
       "Alex Ray",
       "Raul Puri",
       "Gretchen Krueger",
       "Michael Petrov",
       "Heidy Khlaaf",
       "Girish Sastry",
       "Pamela Mishkin",
       "Brooke Chan",
       "Scott Gray",
       "Nick Ryder",
       "Mikhail Pavlov",
       "Alethea Power",
       "Lukasz Kaiser",
       "Mohammad Bavarian",
       "Clemens Winter",
       "Philippe Tillet",
       "Felipe Petroski Such",
       "Dave Cummings",
       "Matthias Plappert",
       "Fotios Chantzis",
       "Elizabeth Barnes",
       "Ariel Herbert-Voss",
       "William Hebgen Guss",
       "Alex Nichol",
       "Alex Paino",
       "Nikolas Tezak",
       "Jie Tang",
       "Igor Babuschkin",
       "Suchir Balaji",
       "Shantanu Jain",
       "William Saunders",
       "Christopher Hesse",
       "Andrew N. Carr",
       "Jan Leike",
       "Josh Achiam",
       "Vedant Misra",
       "Evan Morikawa",
       "Alec Radford",
       "Matthew Knight",
       "Miles Brundage",
       "Mira Murati",
       "Katie Mayer",
       "Peter Welinder",
       "Bob McGrew",
       "Dario Amodei",
       "Sam McCandlish",
       "Ilya Sutskever",
       "Wojciech Zaremba"
     ],
     "year": 2021,
     "venue": "arXiv",
     "arxiv_id": "2107.03374"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The HumanEval evaluation framework is released at https://www.github.com/openai/human-eval. Alignment evaluation data is released at https://github.com/openai/code-align-evals-data. However, the Codex model itself is not released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The HumanEval dataset of 164 hand-written programming problems is released at https://www.github.com/openai/human-eval, as stated in Section 2.2."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is provided. The paper mentions Python and numpy (Figure 3) but does not provide enough detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the evaluation dataset is released, the model weights are not, and there are no scripts or README with commands to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All pass@k results in Tables 1 and 2 are reported as point estimates without confidence intervals or error bars. While the paper develops an unbiased estimator for pass@k (Equation 1, Appendix A), no uncertainty bounds on the estimates are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Comparative claims (e.g., Codex outperforms GPT-J, Codex-S outperforms Codex) are made by comparing raw pass@k numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with full baseline context: Codex-12B solves 28.8% pass@1 vs GPT-3 at 0% and GPT-J at 11.4% (Section 1). Codex-S's improvement over Codex is quantified as 'an average margin of 6.5 percentage points on pass@1 and 15.1 percentage points on pass@100' (Section 4.5). The magnitude of differences is clear throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The HumanEval dataset contains 164 problems but there is no justification for why 164 was chosen, no power analysis, and no discussion of whether this sample size is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as single point estimates. While the unbiased pass@k estimator (Equation 1) accounts for sampling variance mathematically, no standard deviations, error bars, or spread measures across experimental runs are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: GPT-3 (various sizes), GPT-Neo (125M, 1.3B, 2.7B), GPT-J-6B, and TabNine, all evaluated on HumanEval (Table 1). For APPS, GPT-Neo 2.7B fine-tuned results from Hendrycks et al. serve as baseline (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-Neo, GPT-J, and TabNine were all contemporary at time of publication (2021). GPT-J was released in May 2021, the same year as this paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically ablates key components: GPT (no code fine-tuning) vs Codex (code fine-tuned) vs Codex-S (supervised fine-tuned), showing the contribution of each stage. Model size is varied across 8 scales (12M to 12B). Different sampling strategies (random, mean log-prob, back-translation) are compared (Figure 7)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple evaluation metrics are used: pass@1, pass@10, pass@100 (Table 1), BLEU scores (Figure 8), test loss (Figure 4), and the APPS dataset metrics including raw and filtered pass@k (Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Codex-D docstring outputs are graded by hand: 'we grade sample docstrings by hand, considering a docstring correct if it uniquely and accurately specifies the code body. Due to the time consuming nature of this process, we only grade 10 samples per problem, for a total of 1640 problems' (Section 5, Table 3)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval is specifically designed as a held-out test set: 'It is important for these tasks to be hand-written, since our models are trained on a large fraction of GitHub, which already contains solutions to problems from a variety of sources' (Section 2.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down results by model size across all baselines. Table 2 breaks APPS results by difficulty level (Introductory, Interview, Competition). Figure 11 shows performance degradation by chain length. Figure 5 breaks down results by temperature."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Limitations) provides detailed failure analysis: difficulty with long chains of operations (Figure 11), variable binding errors (concrete code example in Section 6), misalignment producing buggy code when prompted with buggy code (Figure 12), and insecure code generation (Figure 15, Appendix G)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: 'we did not observe improvements when starting from a pre-trained language model' (Section 3.2); back-translation ranking 'underperforms mean log-probability ranking' (Section 5); 'choosing the sample based on sum log probability can perform slightly worse than picking randomly' (Section 3.3); misalignment worsens with scale (Figure 12)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: 28.8% pass rate (Table 1), GPT-3 at 0% (Table 1), GPT-J at 11.4% (Table 1), 70.2% with 100 samples (Section 1, consistent with Figure 1 showing Codex-S at 77.5%). The abstract's qualitative claims about limitations are supported by Section 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about fine-tuning improving performance are supported by controlled comparisons: same base model architecture with and without code fine-tuning (GPT vs Codex), and with and without supervised fine-tuning (Codex vs Codex-S). Each comparison varies a single factor. The alignment analysis (Appendix E) carefully distinguishes capability from alignment."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its scope: 'In this work, we focus on the task of generating standalone Python functions from docstrings' (Section 1). The abstract states 'study its Python code-writing capabilities.' While the broader impacts section discusses general code generation, the empirical claims are bounded to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 and Appendix E explicitly distinguish misalignment from incompetence as alternative explanations for model failures. Appendix E.3 considers whether poor performance on buggy prompts could be a robustness failure rather than misalignment. Section 4 discusses data distribution mismatch as a factor."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 explicitly argues for functional correctness over BLEU score as the evaluation metric, and Figure 8 demonstrates empirically that BLEU is a poor proxy for correctness. The paper measures pass@k and claims code generation capability — the measurement and claim are well-aligned with minimal proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model sizes are specified for all models evaluated: Codex at 12M, 25M, 42M, 85M, 300M, 679M, 2.5B, and 12B parameters; GPT-Neo at 125M, 1.3B, 2.7B; GPT-J at 6B (Table 1). Since these are the authors' own models, parameter counts uniquely identify each variant."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows three complete example prompts with the exact format used (header, signature, docstring). Stop sequences are specified: '\\nclass', '\\ndef', '\\n#', '\\nif', '\\nprint' (Section 3.2). Appendix B provides 8 additional full prompt examples. Appendix E.5 shows alignment evaluation prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters are reported: nucleus sampling with top p=0.95, temperatures tested (0.2, 0.4, 0.8), n=200 samples per task. Training: 175-step linear warmup, cosine learning rate decay, 100 billion tokens, Adam with β1=0.9, β2=0.95, ε=10⁻⁸, weight decay 0.1 (Section 3.2)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Codex performs direct model inference from prompts without any tool use, retry logic, or multi-step workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents data preprocessing in detail: collected from 54 million GitHub repos (179 GB unique Python files under 1 MB), filtered out auto-generated files, average line length >100, max line length >1000, low alphanumeric percentage, resulting in 159 GB final dataset. Tokenizer adaptations for whitespace are described (Section 3.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is a dedicated 'Limitations' section discussing specific shortcomings. Additionally, Section 7 (Broader Impacts) provides extensive discussion of risks and limitations across over-reliance, misalignment, bias, security, and economic impacts."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: Codex is 'not sample efficient to train,' struggles with 'docstrings describing long chains of operations' (quantified in Figure 11), and has 'difficulty with binding operations to variables' (concrete code example provided). Appendix E discusses specific alignment threats with empirical evidence."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it focuses on 'generating standalone Python functions from docstrings' (Section 1). Section 6 notes the model 'struggles to parse through increasingly long and higher-level or system-level specifications.' The broader impacts section (7.5) states 'at their current level of capability, Codex models do not materially lower the barrier to entry for malware development.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The HumanEval evaluation dataset is released, but the 159 GB training dataset is not. The training data cannot be independently verified. The model weights are also not released, preventing independent replication of results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection in detail: '54 million public software repositories hosted on GitHub, containing 179 GB of unique Python files under 1 MB,' collected in May 2020. Filtering criteria are specified. Section 4.1-4.2 describe supervised fine-tuning data collection from competitive programming sites and CI-traced projects."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Training data comes from public GitHub repositories. HumanEval problems were hand-written by the authors. The Codex-D docstring grading (Section 5) is done by the authors, not recruited participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: GitHub scraping (179 GB) → filtering by auto-generation, line length, alphanumeric content (159 GB final). For Codex-S training data: competitive programming problems (10,000 curated) + CI-traced functions (~40,000) → quality filtering using Codex-12B to remove ambiguous/stateful problems (Section 4.3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgments section states 'we thank GitHub for partnering to build GitHub Copilot and Microsoft Azure for supporting model training with infrastructure management,' disclosing corporate support for the research."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: '1 OpenAI, San Francisco, California, USA. 2 Anthropic AI, San Francisco, California, USA. Work performed while at OpenAI. 3 Zipline, South San Francisco, California, USA. Work performed while at OpenAI.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI has a direct financial interest in Codex's performance — the paper states 'A distinct production version of Codex powers GitHub Copilot.' OpenAI is evaluating its own commercial product. Microsoft Azure (infrastructure provider) and GitHub (partner for Copilot) also have financial stakes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. The paper does not include a standard conflicts-of-interest declaration, despite OpenAI's commercial interest in Codex through GitHub Copilot."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 'Our training dataset was collected in May 2020 from 54 million public software repositories hosted on GitHub.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly discusses this concern: 'It is important for these tasks to be hand-written, since our models are trained on a large fraction of GitHub, which already contains solutions to problems from a variety of sources. For example, there are more than ten public repositories containing solutions to Codeforces problems, which make up part of the recently proposed APPS dataset.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval was specifically designed to avoid contamination: hand-written problems created after the training data collection (May 2020). The paper notes 'Though not a guarantee for problem novelty, all problems were hand-written and not programmatically copied from existing sources' (Figure 2 caption). For APPS, contamination risk is acknowledged."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates models on benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs are reported. The paper generates 200 samples per problem for 164 problems across multiple model sizes and temperatures, but does not report the total inference cost, tokens consumed, or wall-clock time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.6 states 'The original training of GPT-3-12B consumed hundreds of petaflop/s-days of compute, while fine-tuning it to create Codex-12B consumed a similar amount of compute.' Section 3.2 states training was for 100 billion tokens. The platform (Azure) is identified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Results are generated from a single set of 200 samples per problem with no discussion of how results vary across random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The number of samples is clearly stated: 'we generate n ≥ k samples per task (in this paper, we use n = 200 and k ≤ 100)' (Section 2.1). For APPS evaluation, 1000 samples are generated per task."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While Figure 5 shows pass@k at different temperatures (0.2, 0.4, 0.8), no formal hyperparameter search budget is stated (number of configurations tried, search method, or total compute spent on search)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The temperature selection process is transparent: Figure 5 plots pass@k against temperature for various k values, and the optimal temperature is selected from the upper hull. They report 'the optimal temperature for pass@1 is T*=0.2 and the optimal temperature for pass@100 is T*=0.8' for the 679M model (Section 3.3)."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests with p-values are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their own Codex against GPT-Neo, GPT-J, and TabNine without acknowledging the potential bias of authors evaluating their own system. There is no discussion of how having full control over one system but not others might affect the comparison."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1, 4, and 6 show performance as a function of model size (a proxy for compute). Figure 4 shows test loss follows a power law with model size. Figure 6 shows pass@1 and pass@100 scale as sigmoids in log-parameters."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 provides extensive discussion of construct validity: argues functional correctness is superior to BLEU for measuring code generation, shows empirically that BLEU fails to distinguish correct from incorrect code (Figure 8), and discusses how functional correctness mirrors real software development practice (test-driven development)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All models are evaluated via direct inference with the same prompting approach, so there is no scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Training data was collected in May 2020 (Section 3.1). HumanEval was hand-written specifically for this evaluation and did not exist before the training data collection, inherently addressing temporal leakage. For APPS, the paper acknowledges that GitHub contains Codeforces solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly discuss whether the evaluation setup leaks information through features. For example, function signatures and docstring style in HumanEval may provide implicit cues not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 addresses non-independence by hand-writing HumanEval: 'It is important for these tasks to be hand-written, since our models are trained on a large fraction of GitHub, which already contains solutions to problems from a variety of sources.' This ensures test problems are independent of training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied (no canary strings, membership inference, or n-gram overlap analysis). While hand-writing HumanEval is a prevention strategy, the paper acknowledges 'Though not a guarantee for problem novelty' (Figure 2) and no verification method confirms the problems do not overlap with training data."
       }
     }
   },
   "claims": [
     {
       "claim": "Codex-12B solves 28.8% of HumanEval problems with a single sample, while GPT-3 solves 0% and GPT-J solves 11.4%.",
       "evidence": "Table 1 reports pass@1 for Codex-12B at 28.81%, GPT-Neo models at 0.75-6.41%, and GPT-J at 11.62%. GPT-3 models achieve near 0% (Section 3.4).",
       "supported": "strong"
     },
     {
       "claim": "With 100 samples per problem, Codex-S solves 77.5% of HumanEval problems.",
       "evidence": "Figure 1 shows Codex-S pass@100 at 77.5%. Section 4.5 confirms the result and shows consistent improvements over Codex across model sizes.",
       "supported": "strong"
     },
     {
       "claim": "Supervised fine-tuning on correctly implemented functions (Codex-S) improves pass@1 by an average of 6.5 percentage points and pass@100 by 15.1 percentage points over Codex.",
       "evidence": "Section 4.5 reports these margins averaged across model sizes, with Figure 10 showing the comparison visually.",
       "supported": "strong"
     },
     {
       "claim": "BLEU score is not a reliable indicator of functional correctness for code generation.",
       "evidence": "Figure 8 shows significant overlap between BLEU score distributions of correct and incorrect solutions for 4 HumanEval tasks, demonstrating that functionally inequivalent programs can have higher BLEU scores than correct ones (Section 3.3).",
       "supported": "strong"
     },
     {
       "claim": "Model performance on chained operations degrades exponentially with docstring complexity.",
       "evidence": "Figure 11 shows pass rate dropping by roughly a factor of 2-3 per additional chained component in synthetic tasks (Section 6).",
       "supported": "strong"
     },
     {
       "claim": "Test loss after code fine-tuning follows a power law with model size.",
       "evidence": "Figure 4 shows the power law relationship with functional form (N / 5.92×10^7)^-0.13, closely fitting the empirical data across model sizes (Section 3.3).",
       "supported": "strong"
     },
     {
       "claim": "Mean token log probability is an effective heuristic for selecting the best sample from multiple generations.",
       "evidence": "Figure 7 shows mean log-probability ranking significantly outperforms random selection and back-translation ranking, reaching 44.5% when selecting from 100 samples (Section 3.3).",
       "supported": "strong"
     },
     {
       "claim": "Codex models frequently generate insecure cryptographic code configurations.",
       "evidence": "Figure 15 shows Codex produces insecure RSA keys (<2048 bits) or AES contexts (ECB mode) in a significant fraction of samples across model sizes (Appendix G.3). However, the study covers only two cryptographic scenarios.",
       "supported": "moderate"
     },
     {
       "claim": "When prompted with subtly buggy code, Codex produces worse code than it is capable of, and this gap increases with model size.",
       "evidence": "Figure 12 shows the gap between performance with correct context vs buggy context grows with model size. Adding an instruction to write correct code helps partially but does not eliminate the gap (Appendix E).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Codex, a GPT model fine-tuned on 159 GB of GitHub Python code, solves 28.8% of hand-written HumanEval problems with a single sample, vastly outperforming GPT-3 (0%) and GPT-J (11.4%). Repeated sampling is surprisingly effective: 100 samples per problem yield 77.5% pass rate with Codex-S. The paper establishes functional correctness (pass@k) as the appropriate metric over BLEU, demonstrates power law scaling with model size, and provides extensive analysis of limitations including exponential degradation with docstring complexity, misalignment that worsens with scale, and frequent generation of insecure code.",
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "OpenAI authors evaluate Codex, which powers their commercial product GitHub Copilot. The paper acknowledges the relationship ('A distinct production version of Codex powers GitHub Copilot') but does not include a formal conflicts-of-interest statement. The funder (OpenAI) has direct financial interest in demonstrating strong Codex performance."
     },
     {
       "flag": "No uncertainty quantification on main results",
       "detail": "Despite developing an unbiased estimator for pass@k, the paper reports all main results as point estimates without confidence intervals, error bars, or variance across runs. With 164 problems in HumanEval, the standard error on aggregate pass rates could be substantial."
     },
     {
       "flag": "Training data not released",
       "detail": "The 159 GB training dataset and model weights are not released. While the evaluation dataset (HumanEval) is public, the core claims about Codex's performance cannot be independently replicated because neither the training data nor the model is available."
     }
   ],
   "cited_papers": [
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": [
         "Brown, T. B.",
         "Mann, B.",
         "Ryder, N.",
         "et al."
       ],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "Foundation GPT-3 model that Codex is fine-tuned from; baseline for code generation capability comparison."
     },
     {
       "title": "Measuring Coding Challenge Competence with APPS",
       "authors": [
         "Hendrycks, D.",
         "Basart, S.",
         "Kadavath, S.",
         "et al."
       ],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Coding challenge benchmark used to evaluate Codex alongside HumanEval; measures functional correctness on competitive programming tasks."
     },
     {
       "title": "GPT-J-6B: A 6 Billion Parameter Autoregressive Language Model",
       "authors": [
         "Wang, B.",
         "Komatsuzaki, A."
       ],
       "year": 2021,
       "relevance": "Open-source language model baseline for code generation, trained on The Pile with 8% GitHub code."
     },
     {
       "title": "GPT-Neo: Large Scale Autoregressive Language Modeling with Mesh-Tensorflow",
       "authors": [
         "Black, S.",
         "Gao, L.",
         "Wang, P.",
         "Leahy, C.",
         "Biderman, S."
       ],
       "year": 2021,
       "relevance": "Open-source GPT-style model serving as baseline for code generation; trained on The Pile."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": [
         "Feng, Z.",
         "Guo, D.",
         "Tang, D.",
         "et al."
       ],
       "year": 2020,
       "relevance": "Pre-trained code representation model using BERT objective on docstring-function pairs."
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": [
         "Kaplan, J.",
         "McCandlish, S.",
         "Henighan, T.",
         "et al."
       ],
       "year": 2020,
       "arxiv_id": "2003.05950",
       "relevance": "Establishes power law scaling relationships that Codex's performance also follows after code fine-tuning."
     },
     {
       "title": "SPoC: Search-based Pseudocode to Code",
       "authors": [
         "Kulal, S.",
         "Pasupat, P.",
         "Chandra, K.",
         "et al."
       ],
       "year": 2019,
       "relevance": "Introduced the pass@k metric for evaluating functional correctness of synthesized code."
     },
     {
       "title": "Unsupervised Translation of Programming Languages",
       "authors": [
         "Lachaux, M.-A.",
         "Rozière, B.",
         "Chanussot, L.",
         "Lample, G."
       ],
       "year": 2020,
       "arxiv_id": "2006.03511",
       "relevance": "Demonstrated functional correctness as a better evaluation metric than BLEU for code translation."
     },
     {
       "title": "Extracting Training Data from Large Language Models",
       "authors": [
         "Carlini, N.",
         "Tramèr, F.",
         "Wallace, E.",
         "et al."
       ],
       "year": 2021,
       "relevance": "Demonstrates privacy risks of training data memorization in large language models, applicable to code models trained on public repositories."
     },
     {
       "title": "You Autocomplete Me: Poisoning Vulnerabilities in Neural Code Completion",
       "authors": [
         "Schuster, R.",
         "Song, C.",
         "Tromer, E.",
         "Shmatikov, V."
       ],
       "year": 2020,
       "relevance": "Demonstrates data poisoning attacks on code completion models, a supply chain security risk for code generation."
     },
     {
       "title": "In-IDE Code Generation from Natural Language: Promise and Challenges",
       "authors": [
         "Xu, F. F.",
         "Vasilescu, B.",
         "Neubig, G."
       ],
       "year": 2021,
       "arxiv_id": "2101.11149",
       "relevance": "Evaluates capabilities and challenges of code generation in IDE settings, directly relevant to code generation evaluation."
     },
     {
       "title": "Learning Autocompletion from Real-World Datasets",
       "authors": [
         "Aye, G. A.",
         "Kim, S.",
         "Li, H."
       ],
       "year": 2021,
       "relevance": "Reports on Facebook's internal code autocomplete tool, providing industry perspective on code generation deployment."
     },
     {
       "title": "The Pile: An 800GB Dataset of Diverse Text for Language Modeling",
       "authors": [
         "Gao, L.",
         "Biderman, S.",
         "Black, S.",
         "et al."
       ],
       "year": 2020,
       "relevance": "Training dataset for GPT-Neo and GPT-J baselines, containing 8% GitHub code that enables programming capabilities."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Codex directly powers GitHub Copilot, a tool millions of developers use daily, and the paper introduces HumanEval which became a standard benchmark."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that repeated sampling (100 samples) jumps from 28.8% to 77.5% was genuinely surprising and counterintuitive to most practitioners."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Extensive analysis of insecure code generation, misalignment that worsens with scale, and potential for malware generation makes safety a major theme."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "OpenAI evaluating its own commercial product raises mild conflict, but the paper is more celebratory than controversial."
     },
     "demo_ability": {
       "score": 2,
       "justification": "HumanEval benchmark is publicly released and Codex was available via API, though the model weights and training data were not released."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "From OpenAI, powers GitHub Copilot used by millions, authored by figures including Dario Amodei, Sam McCandlish, and Ilya Sutskever."
     }
   }
 }