ai-research-survey

scan.json (18703B)

---

 {
   "paper": {
     "title": "ComBack: A Versatile Dataset for Enhancing Compiler Backend Development Efficiency",
     "authors": ["Ming Zhong", "Fang Lyu", "Lulin Wang", "Hongna Geng", "Lei Qiu", "Huimin Cui", "Xiaobing Feng"],
     "year": 2024,
     "venue": "NeurIPS 2024 Track on Datasets and Benchmarks"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Fine-tuned models and code are available at https://huggingface.co/docz1105/ComBack_Models (Sec. 4.1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset is publicly available at https://huggingface.co/datasets/docz1105/ComBack (abstract)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (64-core Intel Xeon Gold CPU, 8 NVIDIA Tesla V100 GPUs) but no requirements.txt, Dockerfile, or detailed software environment specification is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or scripts to replicate experiments are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., EM percentages, ED scores) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CodeT5+ 'outperformed' and 'surpassed' baselines based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute differences are reported with baseline context, e.g., 'CodeT5+ surpasses 37.10%-40.82% for EM compared with ChatGPT' and improvements of '41.64-77.21 of ED across three tasks' (Sec. 4.2, 4.3)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 6 models were chosen as the sample, or why 3 targets were selected for new-target experiments. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or any spread measures are reported. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Fork-Flow (conventional method), ChatGPT-3.5-Turbo, and Code-LLaMA-34B-Instruct (Sec. 4.1, Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ChatGPT-3.5-Turbo and Code-LLaMA-34B-Instruct were contemporary large language models at the time of submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 4.3.2 ablates by removing GPU/MPU data (training on CPU only) and Sec. 4.4 ablates by adding/removing RISC-V data, showing component contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: Exact Match (EM), Edit Distance Similarity (ED), and BLEU-4 across different tasks (Sec. 4.1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality. All evaluation is automated via EM, ED, and BLEU-4 metrics. For a code generation dataset, human evaluation of code correctness/usefulness would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Data is split into train/validation/test sets (80%:10%:10% in Sec. 4.2, 85%:15% train/val with separate test targets in Sec. 4.3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per target (RISC-V, ARC, NVPTX) in Tables 3-5, per compiler (GCC vs LLVM), and per task."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 4.3.2 and Appendix H discuss accuracy decreases for new target types and analyze why the model struggles with certain functions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows accuracy decreases when training without GPU/MPU data, and the paper acknowledges limitations in handling customized functions (Sec. 6 Discussion)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about fine-tuned CodeT5+ outperforming Fork-Flow, ChatGPT, and Code-LLaMA are supported by Tables 2-3 and Figure 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('ComBack effectively improves...') are supported by before/after fine-tuning comparisons (Table 2) and controlled ablations (Tables 4-5), which constitute adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to GCC and LLVM compiler backends. The paper specifies exact targets tested and acknowledges limitations for new target types (Sec. 4.3.2)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, whether the improvements stem from domain-specific vocabulary rather than structural understanding, or whether the comparison with ChatGPT/Code-LLaMA is unfair due to different input formats."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "ChatGPT-3.5-Turbo is mentioned without a snapshot date. Code-LLaMA-34B-Instruct is specified by size. CodeT5+-220M is specified. No API version dates for ChatGPT."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F provides the prompts used for ChatGPT and Code-LLaMA experiments (referenced in Sec. 4.3.1)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 in Appendix C reports hyperparameters: training batch size 32, evaluation batch size 16, beam size 4, learning rate 5e-5, max optimization steps 3, and sequence length settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are fine-tuned and used for direct inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 3.2 documents four preprocessing steps in detail: code collection, function description collection, code extraction with tree-sitter, and target-specific value extraction with intermediate representations."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 6 (Discussion) contains a 'Limitation' subsection noting the absence of function descriptions for highly-customized functions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitation mentioned is specific but very narrow (missing function descriptions). No discussion of threats like data leakage between train/test splits sharing the same compiler infrastructure, or whether EM/BLEU adequately measure practical usefulness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper does not clarify, for example, that matching code tokens does not mean the generated code compiles or is semantically correct."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is publicly available on HuggingFace (https://huggingface.co/datasets/docz1105/ComBack), enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 3.2 describes crawling GitHub with specific keywords, collecting from official GCC/LLVM websites, version ranges (GCC 3.0-13.0, LLVM 2.0.1-17.0.1), and filtering steps."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is open-source compiler code from GitHub and official websites."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 3.2 documents the full pipeline: collection → duplicate removal → function extraction via tree-sitter → target-specific value extraction. Sample counts are given for each task (161,124; 216,315; 45,296)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgement section lists National Key R&D Program of China, Strategic Priority Research Program of CAS, National Natural Science Foundation of China, and ICT Innovation Funding with grant numbers."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: SKLP, Institute of Computing Technology, CAS, and UCAS, Beijing, China."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Chinese government research agencies (NSFC, CAS) with no apparent financial stake in compiler backend tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares against ChatGPT-3.5-Turbo and Code-LLaMA but does not state their training data cutoff dates. It's unclear whether these models may have seen compiler backend code in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ChatGPT or Code-LLaMA may have been trained on the same GCC/LLVM source code used in the test sets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GCC and LLVM source code is widely available online. The paper does not discuss whether baseline LLMs may have seen this code during pre-training, which could affect the comparison."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported for any of the models despite comparing a 220M parameter model against much larger ones."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (8 NVIDIA Tesla V100 GPUs) but no training time, GPU hours, or total compute budget is stated."
       }
     }
   },
   "claims": [
     {
       "claim": "ComBack effectively improves backend development capabilities of six language models across three tasks, with ED improvements of 41.64-77.21 after fine-tuning.",
       "evidence": "Table 2 shows before/after fine-tuning accuracy for all six models across three tasks (Sec. 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuned CodeT5+ (220M) outperforms ChatGPT-3.5-Turbo and Code-LLaMA-34B-Instruct on compiler backend tasks.",
       "evidence": "Table 3 shows CodeT5+ surpasses ChatGPT by 37-41% EM and Code-LLaMA by 50-55% EM on statement-level completion (Sec. 4.3.1).",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuned CodeT5+ achieves higher accuracy than Fork-Flow conventional development methods for code generation.",
       "evidence": "Figure 6 compares CodeT5+ against Fork-Flow average and maximum edit distance/BLEU-4 scores (Sec. 4.3.1).",
       "supported": "moderate"
     },
     {
       "claim": "ComBack supports iterative expansion: adding RISC-V data improves accuracy for RI5CY by +7.90% EM on statement completion and +25.05 BLEU-4 on code generation.",
       "evidence": "Table 5 shows accuracy improvements across all three tasks after adding RISC-V to training data (Sec. 4.4).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ComBack is the first public dataset for compiler backend development, covering 178 backends across GCC and LLVM with three tasks (statement completion, next-statement suggestion, code generation). Fine-tuning a small 220M-parameter CodeT5+ model on ComBack significantly outperforms ChatGPT-3.5-Turbo and Code-LLaMA-34B-Instruct on all three tasks. The dataset demonstrates cross-target generalization: models trained on CPU backends can generate code for MPU and GPU backends, and iterative data expansion improves accuracy for customized targets.",
   "red_flags": [
     {
       "flag": "No variance or multiple-run reporting",
       "detail": "All results are single-run point estimates with no standard deviations, confidence intervals, or indication of result stability across random seeds."
     },
     {
       "flag": "No functional correctness evaluation",
       "detail": "Evaluation uses token-level metrics (EM, ED, BLEU-4) only. No assessment of whether generated code compiles, passes tests, or is semantically correct. High BLEU-4 does not guarantee functional code."
     },
     {
       "flag": "Potentially unfair baseline comparison",
       "detail": "ChatGPT and Code-LLaMA are used zero-shot on a highly specialized domain (compiler backends), while CodeT5+ is fine-tuned on domain-specific data. The comparison demonstrates the value of fine-tuning but may overstate CodeT5+'s relative capability."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "GCC and LLVM source code is widely available on GitHub. The baseline LLMs (ChatGPT, Code-LLaMA) may have seen this code during pre-training, making the comparison results harder to interpret."
     }
   ],
   "cited_papers": [
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière"],
       "year": 2024,
       "relevance": "Major open-source code LLM used as a baseline, directly relevant to LLM code generation capability evaluation."
     },
     {
       "title": "CodeT5+: Open Code Large Language Models for Code Understanding and Generation",
       "authors": ["Yue Wang"],
       "year": 2023,
       "relevance": "Best-performing model in the study, representative of encoder-decoder code LLMs."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng"],
       "year": 2020,
       "relevance": "Foundational pre-trained code model used as one of six fine-tuned models in the evaluation."
     },
     {
       "title": "GraphCodeBERT: Pre-training code representations with data flow",
       "authors": ["Daya Guo"],
       "year": 2021,
       "relevance": "Code representation model incorporating data flow, evaluated as a baseline in the study."
     },
     {
       "title": "UniXcoder: Unified cross-modal pre-training for code representation",
       "authors": ["Daya Guo"],
       "year": 2022,
       "relevance": "Cross-modal code model evaluated in the study, relevant to code understanding and generation benchmarks."
     },
     {
       "title": "CodeXGlue: A machine learning benchmark dataset for code understanding and generation",
       "authors": ["Shuai Lu"],
       "year": 2021,
       "relevance": "Influential code benchmark dataset whose methodology ComBack follows for data extraction."
     },
     {
       "title": "NatGen: Generative pre-training by 'naturalizing' source code",
       "authors": ["Saikat Chakraborty"],
       "year": 2022,
       "relevance": "Code generation pre-training approach evaluated as one of six models in the study."
     },
     {
       "title": "Large Language Models for Compiler Optimization",
       "authors": ["Chris Cummins"],
       "year": 2023,
       "arxiv_id": "2309.07062",
       "relevance": "Directly relevant work applying LLMs to compiler tasks, showing AI potential for compilation."
     }
   ]
 }