ai-research-survey

scan.json (19756B)

---

 {
   "paper": {
     "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
     "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang", "Nan Duan", "Xiaocheng Feng", "Ming Gong", "Linjun Shou", "Bing Qin", "Ting Liu", "Daxin Jiang", "Ming Zhou"],
     "year": 2020,
     "venue": "EMNLP 2020 Findings",
     "arxiv_id": "2002.08155"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All the codes and data are available at https://github.com/microsoft/CodeBERT' (footnote 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use the publicly available CodeSearchNet Corpus and release their NL-PL probing dataset. The GitHub repo contains data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (NVIDIA DGX-2 with V100 GPUs) and some hyperparameters but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides hyperparameters in Appendix B but no step-by-step reproduction instructions or README with commands in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., MRR values in Table 2, BLEU scores in Tables 3-5) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CodeBERT 'achieves state-of-the-art performance' based on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw metric values are reported but no standardized effect sizes (Cohen's d, etc.). Differences are stated numerically (e.g., '1.3 BLEU score over RoBERTa') but without formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why the CodeSearchNet dataset sizes or probing dataset sizes are appropriate. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or multiple-run results are reported. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: joint embeddings (NBoW, CNN, BiRNN, SelfAtt), RoBERTa, RoBERTa pre-trained with code only, and task-specific baselines for code summarization (Table 2, 3, 4, 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include RoBERTa (2019), ELECTRA-style RTD (2020), code2seq (2019), and the CodeSearchNet baselines which were contemporary at submission time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 ablates initialization (from scratch vs. RoBERTa) and training objectives (MLM only, RTD only, MLM+RTD). Section 4.1 discusses these systematically."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Code search uses MRR across 6 languages. Code summarization uses smoothed BLEU-4. Probing uses accuracy. Multiple tasks with appropriate metrics for each."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. Code search is evaluated by MRR and code summarization by BLEU only. Qualitative examples are shown in Appendix E but not systematically evaluated by humans."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses separate train/dev/test splits from CodeSearchNet (Table 6) and reports results on test sets. Models are selected based on dev set performance (Appendix B.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per programming language (6 languages for code search in Table 2, 6 languages for summarization in Table 3/4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses that CodeBERT achieves 'slightly lower results than code2seq' on C# and explains the likely reason (lack of AST information). Section 4.2 discusses probing limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that training CodeBERT by traversing AST structure 'does not bring improvements on generation tasks' (Section 4.4), and that CodeBERT underperforms code2seq on C#."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims state-of-the-art on code search and code documentation generation, which are supported by Tables 2-4. The probing claims are supported by Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about which components help are supported by ablation studies in Table 2 (different objectives and initialization strategies). The ablation design is controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Programming and Natural Languages' broadly, but results are limited to 6 languages for code search and 1 additional language (C#) for generalization. The broad title overstates the tested scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. The paper does not consider confounds such as whether improvements come from more data, different tokenization, or other factors beyond the proposed objectives."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes CodeBERT as having 'the same model architecture as RoBERTa-base' (125M parameters) but does not specify exact RoBERTa checkpoint version used for initialization."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting. It is a pre-training + fine-tuning approach."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters are provided in Appendix B: batch size, learning rate, optimizer, warmup steps, max length, training steps for pre-training (B.1), code search fine-tuning (B.2), and summarization (B.3, B.4)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard pre-train + fine-tune approach."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data sources (CodeSearchNet for bimodal, additional GitHub repos for unimodal), filtering criteria, and data statistics are in Table 6 and Appendix A."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion mentions future directions but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The CodeSearchNet Corpus is publicly available, and the probing dataset is released via GitHub. Raw data can be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection: bimodal data from CodeSearchNet (NL-PL pairs from GitHub), unimodal data from GitHub repos with at least one star. Probing dataset construction is described in Section 4.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from public GitHub repositories and CodeSearchNet benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the pipeline: CodeSearchNet provides bimodal data across 6 languages, unimodal data from GitHub with filtering (repos with ≥1 star), and data statistics are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists National Key R&D Program of China (2018YFB1005103) and NSFC grants (61632011, 61772156)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Harbin Institute of Technology, Sun Yat-sen University, Microsoft Research Asia, and Microsoft Search Technology Center Asia."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Government grants (Chinese national programs and NSFC) are independent of the outcome. However, Microsoft authors evaluating a Microsoft-released model is a potential conflict not explicitly addressed, though this question specifically asks about funders."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Several authors are Microsoft employees evaluating a Microsoft-released model, but no financial interest disclosure is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state when the CodeSearchNet or unimodal training data was collected, nor does it discuss the temporal relationship between training and test data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between pre-training data (GitHub repos) and downstream evaluation data (also from GitHub). The probing dataset is constructed from the same CodeSearchNet data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The pre-training uses GitHub code which could overlap with CodeSearchNet test sets. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported for any of the downstream tasks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 states pre-training was done on one NVIDIA DGX-2 with 16 V100 GPUs using FP16, and provides timing: '1,000 batches costs 600 minutes with MLM objective, 120 minutes with RTD objective.'"
       }
     }
   },
   "claims": [
     {
       "claim": "CodeBERT achieves state-of-the-art performance on natural language code search across six programming languages.",
       "evidence": "Table 2 shows CodeBERT (MLM+RTD, INIT=R) achieves the highest MRR on all six languages compared to baselines including RoBERTa and joint embedding methods.",
       "supported": "strong"
     },
     {
       "claim": "CodeBERT achieves state-of-the-art performance on code-to-documentation generation.",
       "evidence": "Tables 3 and 4 show CodeBERT outperforms RoBERTa and code-only pre-trained models on smoothed BLEU-4 across six languages.",
       "supported": "strong"
     },
     {
       "claim": "The replaced token detection (RTD) objective combined with MLM improves over MLM alone.",
       "evidence": "Table 2 ablation shows MLM+RTD consistently outperforms MLM-only across languages. The improvement is modest but consistent.",
       "supported": "moderate"
     },
     {
       "claim": "CodeBERT learns better NL-PL representations than RoBERTa in zero-shot probing.",
       "evidence": "Table 1 shows CodeBERT achieves higher accuracy than RoBERTa on NL and PL probing tasks in a zero-shot setting with frozen parameters.",
       "supported": "strong"
     },
     {
       "claim": "CodeBERT generalizes to programming languages not seen during pre-training (C#).",
       "evidence": "Table 5 shows CodeBERT achieves 22.36 BLEU on C# summarization, improving over RoBERTa (19.81), though below code2seq (23.04).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeBERT is a bimodal pre-trained model for programming and natural languages, trained with masked language modeling and replaced token detection objectives on CodeSearchNet data. It achieves state-of-the-art results on natural language code search (MRR) and code documentation generation (BLEU-4) across six programming languages. The model also demonstrates transfer to unseen languages (C#) and shows superior zero-shot probing performance compared to RoBERTa, suggesting it learns meaningful NL-PL alignment.",
   "red_flags": [
     {
       "flag": "No variance or significance testing",
       "detail": "All results are single-run point estimates with no error bars, standard deviations, or significance tests. Differences between models may not be statistically significant."
     },
     {
       "flag": "Potential contamination between pre-training and evaluation data",
       "detail": "Both pre-training data and evaluation benchmarks come from GitHub. The paper does not discuss whether test set code appeared in pre-training data."
     },
     {
       "flag": "Microsoft employees evaluating Microsoft model",
       "detail": "Multiple authors are from Microsoft Research Asia and Microsoft Search Technology Center. CodeBERT is a Microsoft product released on Microsoft's GitHub. No conflict of interest statement is provided."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity discussion despite several methodological gaps (single runs, contamination risk, limited language coverage)."
     }
   ],
   "cited_papers": [
     {
       "title": "CodeSearchNet Challenge: Evaluating the State of Semantic Code Search",
       "authors": ["Hamel Husain", "Ho-Hsiang Wu", "Tiferet Gazit", "Miltiadis Allamanis", "Marc Brockschmidt"],
       "year": 2019,
       "arxiv_id": "1909.09436",
       "relevance": "Major benchmark dataset for code search that CodeBERT is evaluated on; foundational for NL-PL evaluation methodology."
     },
     {
       "title": "ELECTRA: Pre-training Text Encoders as Discriminators Rather than Generators",
       "authors": ["Kevin Clark", "Minh-Thang Luong", "Quoc V. Le", "Christopher D. Manning"],
       "year": 2020,
       "relevance": "Introduces the replaced token detection objective that CodeBERT adapts for bimodal NL-PL pre-training."
     },
     {
       "title": "RoBERTa: A Robustly Optimized BERT Pretraining Approach",
       "authors": ["Yinhan Liu", "Myle Ott", "Naman Goyal"],
       "year": 2019,
       "arxiv_id": "1907.11692",
       "relevance": "Primary baseline model and initialization source for CodeBERT; key comparison point for measuring the value of bimodal pre-training."
     },
     {
       "title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding",
       "authors": ["Jacob Devlin", "Ming-Wei Chang", "Kenton Lee", "Kristina Toutanova"],
       "year": 2018,
       "arxiv_id": "1810.04805",
       "relevance": "Foundational pre-training approach that CodeBERT extends to the bimodal NL-PL setting."
     },
     {
       "title": "code2seq: Generating Sequences from Structured Representations of Code",
       "authors": ["Uri Alon", "Shaked Brody", "Omer Levy", "Eran Yahav"],
       "year": 2019,
       "relevance": "Key baseline for code summarization that outperforms CodeBERT on C#, demonstrating value of AST-based representations."
     },
     {
       "title": "Summarizing Source Code Using a Neural Attention Model",
       "authors": ["Srinivasan Iyer", "Ioannis Konstas", "Alvin Cheung", "Luke Zettlemoyer"],
       "year": 2016,
       "relevance": "Introduces CodeNN and the StackOverflow C# dataset used for evaluating CodeBERT's generalization to unseen languages."
     },
     {
       "title": "Deep Code Search",
       "authors": ["Xiaodong Gu", "Hongyu Zhang", "Sunghun Kim"],
       "year": 2018,
       "relevance": "Early neural code search approach and baseline for evaluating CodeBERT on code retrieval tasks."
     },
     {
       "title": "Pre-trained Contextual Embedding of Source Code",
       "authors": ["Aditya Kanade", "Petros Maniatis", "Gogul Balakrishnan", "Kensen Shi"],
       "year": 2019,
       "arxiv_id": "2001.00059",
       "relevance": "Concurrent work on pre-training for code (CuBERT), relevant to understanding the landscape of code pre-training models."
     }
   ]
 }