ai-research-survey

scan.json (17298B)

---

 {
   "paper": {
     "title": "Compilable Neural Code Generation with Compiler Feedback",
     "authors": ["Xin Wang", "Yasheng Wang", "Yao Wan", "Fei Mi", "Yitong Li", "Pingyi Zhou", "Jin Liu", "Hao Wu", "Xin Jiang", "Qun Liu"],
     "year": 2022,
     "venue": "ACL 2022 (Findings)",
     "arxiv_id": "2203.05132"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: CodeSearchNet-Python and AdvTest from CodeXGLUE. The filtering criteria to extract subsets are described."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python 3 and NVIDIA Tesla V100 GPUs but does not provide a requirements.txt, Dockerfile, or detailed dependency list."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims COMPCODER outperforms baselines but provides no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Improvements are reported with baseline context, e.g., 'improving the success rate of compilation from 44.18 to 89.18' and absolute point differences are stated."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of 50k/45k/5k split for code completion or 41k/40k/1k for text-to-code."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multiple-run results are reported. All numbers appear to be single-run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: BiLSTM, Transformer, GPT-2, CodeGPT, PLBART, CodeT5 (Tables 1 and 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CodeGPT, PLBART, and CodeT5 were all recent state-of-the-art models at the time of writing (2021-2022)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents an ablation study examining RL, Dtrain, and Dtest components individually and in combination."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: Edit Similarity (ES) and Compilation Rate (CR)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. All evaluation is automated via compiler checks and edit similarity."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Separate test sets are used: 5k for code completion, 1k for text-to-code generation (Section 4.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Code completion results are broken down by token count (25, 30, 35, 40, 45 tokens) in Table 1 and Figure 4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows failure cases where candidates fail to compile, and Figure 5 shows case studies comparing CodeGPT failures with COMPCODER successes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results or failed approaches are reported. Every experiment shows improvement."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of improving compilation rate from 44.18 to 89.18 (code completion) and 70.3 to 96.2 (text-to-code) are supported by Tables 1 and 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by the ablation study (Table 3), which uses controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Neural Code Generation' broadly but experiments are only on Python. No explicit statement bounding generalization to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. The weak constraint issue (whitespace strings compiling) is noted but no broader alternatives are considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The specific CodeGPT checkpoint is identified via HuggingFace URL: 'microsoft/CodeGPT-small-py-adaptedGPT2' (Section 4.4)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper fine-tunes models rather than using prompting. No prompts are used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 reports learning rate (1.5e-5), batch size (32/16), max epochs (20), sequence lengths, beam size (5), and RL data sampling (5%)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a training pipeline, not an agent system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes filtering: extracting compilable Python 3 methods with token lengths 64-96 from CodeSearchNet, and filtering AdvTest for code lengths 128-170 and text lengths >5."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. The conclusion briefly mentions compilability doesn't guarantee correctness."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statement of what the results do NOT show. The paper does not bound its claims to Python or the specific datasets tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying datasets (CodeSearchNet, AdvTest/CodeXGLUE) are publicly available, though the specific filtered subsets are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes how data was collected: extracting from CodeSearchNet with length filters, and from AdvTest with length and semantic filters."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data comes from standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the pipeline: source dataset → language/version filter → length filter → compilability filter → train/test split, with counts at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists NSFC grants No. 61972290, 62102157, and 61962061."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed, including Huawei Noah's Ark Lab and Huawei Technologies Co., Ltd."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSFC (Chinese national science foundation), which has no stake in the specific outcomes of this code generation research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Several authors are from Huawei, which has commercial interest in code generation, but this is not explicitly acknowledged as a conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The CodeGPT model's training data cutoff is not stated. The paper uses pre-trained CodeGPT on benchmarks without discussing when its training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CodeSearchNet or AdvTest data appeared in CodeGPT's pre-training corpus."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeSearchNet was published in 2019 and CodeGPT was pre-trained on code corpora that likely included it. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported despite the multi-stage pipeline involving beam search and discriminator evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (2 NVIDIA Tesla V100 32GB) but total training time and compute budget are not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "COMPCODER improves average compilation rate from 44.18 to 89.18 in code completion compared to CodeGPT",
       "evidence": "Table 1 and Figure 4 show compilation rates across different token completion lengths (25-45 tokens). Average improvement is ~45 points.",
       "supported": "strong"
     },
     {
       "claim": "COMPCODER improves compilation rate from 70.3 to 96.2 in text-to-code generation compared to CodeGPT",
       "evidence": "Table 2 shows CR of 96.2 for COMPCODER vs 70.3 for CodeGPT on the AdvTest dataset.",
       "supported": "strong"
     },
     {
       "claim": "COMPCODER does not sacrifice code fluency (Edit Similarity) while improving compilation rate",
       "evidence": "Tables 1 and 2 show ES scores are comparable or slightly better (64.53 vs 64.47 in completion, 62.74 vs 61.82 in text-to-code).",
       "supported": "moderate"
     },
     {
       "claim": "All three components (RL, Dtrain, Dtest) contribute to compilation rate improvement",
       "evidence": "Table 3 ablation study shows incremental improvements: CodeGPT 46.84 → +Dtrain 64.88 → +RL 76.48 → +RL+Dtrain 83.14 → full model 94.48.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "COMPCODER, a three-stage pipeline using compiler feedback (fine-tuning, RL-based compilability reinforcement, and compilability discrimination), dramatically improves compilation rates of generated code. On Python code completion, it raises compilation rate from 44.18% to 89.18% vs CodeGPT, and on text-to-code generation from 70.3% to 96.2%, without degrading edit similarity. An ablation study confirms all three components contribute, with the discriminator at inference time providing the largest single boost.",
   "red_flags": [
     {
       "flag": "No variance or multiple runs",
       "detail": "All results appear to be single-run with no error bars, standard deviations, or confidence intervals, making it impossible to assess result stability."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. Only a brief mention in the conclusion that compilability does not equal correctness."
     },
     {
       "flag": "Potential contamination risk",
       "detail": "CodeGPT was pre-trained on code corpora that likely include CodeSearchNet data. The paper does not discuss whether training/test overlap could inflate results."
     },
     {
       "flag": "Python-only evaluation with broad claims",
       "detail": "The title and framing suggest general 'neural code generation' but all experiments are on Python only. No discussion of generalizability to other languages."
     }
   ],
   "cited_papers": [
     {
       "title": "CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation",
       "authors": ["Shuai Lu", "Daya Guo", "Shuo Ren"],
       "year": 2021,
       "relevance": "Major code generation benchmark used in this paper's evaluation, widely used in LLM code generation research."
     },
     {
       "title": "Unified Pre-training for Program Understanding and Generation",
       "authors": ["Wasi Uddin Ahmad", "Saikat Chakraborty", "Baishakhi Ray", "Kai-Wei Chang"],
       "year": 2021,
       "relevance": "PLBART, a pre-trained code model used as a baseline, relevant to understanding code generation model evolution."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Yue Wang"],
       "year": 2021,
       "relevance": "CodeT5 baseline model, influential in pre-trained code generation models."
     },
     {
       "title": "Sequencer: Sequence-to-Sequence Learning for End-to-End Program Repair",
       "authors": ["Zimin Chen", "Steve Kommrusch", "Michele Tufano"],
       "year": 2021,
       "relevance": "Program repair using deep learning, relevant to automated code generation and repair quality."
     },
     {
       "title": "Energy-based Models for Code Generation under Compilability Constraints",
       "authors": ["Tomasz Korbak", "Hady ElSahar", "Marc Dymetman", "German Kruszewski"],
       "year": 2021,
       "arxiv_id": "2106.04985",
       "relevance": "Directly related work on compilability-constrained code generation using energy-based models."
     },
     {
       "title": "SPoC: Search-based Pseudocode to Code",
       "authors": ["Sumith Kulal", "Panupong Pasupat", "Kartik Chandra"],
       "year": 2019,
       "relevance": "Introduced compilation rate as an evaluation metric for code generation."
     },
     {
       "title": "Retrieval Augmented Code Generation and Summarization",
       "authors": ["Md. Rizwan Parvez", "Wasi Uddin Ahmad"],
       "year": 2021,
       "relevance": "Retrieval-augmented approach to code generation, relevant to AI-assisted programming."
     },
     {
       "title": "CodeSearchNet Challenge: Evaluating the State of Semantic Code Search",
       "authors": ["Hamel Husain", "Ho-Hsiang Wu"],
       "year": 2019,
       "relevance": "Source dataset used in this paper's code completion experiments, widely used code benchmark."
     }
   ]
 }