ai-research-survey

scan.json (20485B)

---

 {
   "paper": {
     "title": "The New Compiler Stack: A Survey on the Synergy of LLMs and Compilers",
     "authors": ["Shuoming Zhang", "Jiacheng Zhao", "Qiuchu Yu", "Chunwei Xia", "Zheng Wang", "Xiaobing Feng", "Huimin Cui"],
     "year": 2026,
     "venue": "CCF Transactions on High Performance Computing",
     "arxiv_id": "2601.02045",
     "doi": "10.1007/s42514-025-00270-x"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This systematic literature review of 159 papers proposes a multi-dimensional taxonomy for LLM-enabled compilation, categorizing work by Design Philosophy (Selector, Translator, Generator), LLM Methodology (Training-Required vs Training-Free), Level of Code Abstraction, and Task Type. The survey identifies three primary advancements: democratization of compiler development, discovery of novel optimization strategies, and broadening of compiler scope. Key challenges identified include correctness/verifiability, scalability, interpretability, and cost-effectiveness, with hybrid compiler systems seen as the most promising path forward.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code release, or analysis scripts are provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The curated corpus of 159 papers is not released as a structured dataset. No supplementary data files are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tool specifications are provided for reproducing the survey methodology."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the literature search and selection process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not run experiments or produce statistical results."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments requiring significance testing."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental comparisons requiring effect sizes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no experimental sample size to justify."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments requiring variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare against prior surveys or reviews in a structured way. While it references prior work, it does not systematically compare its coverage, methodology, or findings against existing surveys."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No structured comparison with prior surveys is made, so contemporaneity of baselines cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system to evaluate with metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; human evaluation of system outputs is not applicable."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides detailed per-category breakdowns across multiple dimensions: Design Philosophy (Selector/Translator/Generator), LLM Methodology, Level of Code Abstraction, and Task Type, with Tables 2-4 summarizing papers by task and code level."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses common challenges including correctness failures, scalability limitations, and interpretability issues. The paper notes that neural compilation 'fails to outperform existing compiler systems' and discusses where LLM approaches break down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that early neural compilation work achieved 0% accuracy for optimized settings (§5.2), that LLM-based compilation 'is still preliminary' and 'cannot surpass traditional compilers', and that there are more high-level studies than low-level because low-level code is less amenable to LLMs."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (taxonomy proposal, three primary benefits, correctness/scalability challenges, hybrid systems as promising direction) are all substantiated by the detailed analysis in Sections 3-6."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The survey makes descriptive/taxonomic claims about the field rather than causal claims about specific interventions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 explicitly defines the scope, distinguishing LLM-based approaches from traditional ML techniques and purely NLP-based SE tasks. The paper is bounded to 'LLM-enabled compilation' with clear inclusion/exclusion criteria."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "As a taxonomy/survey paper with no empirical results of its own, alternative explanations for observed results are not applicable."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no measurements of its own."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use any models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental setup requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the three-phase literature search and selection protocol with counts at each stage (311 → 246 → 159), along with explicit inclusion/exclusion criteria for filtering."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The paper discusses challenges of the field (§6.2) but does not discuss limitations of its own survey methodology."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to the validity of the survey itself are discussed. The paper does not address potential biases in its literature search, selection criteria, or taxonomy design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 explicitly defines scope boundaries, stating what is excluded: traditional ML techniques, purely NLP-based SE tasks, and studies not involving Transformer-based LLMs for compilation-related tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The list of 159 papers is not provided as a downloadable dataset. Tables 2-4 list many but the full corpus is not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the data collection: searches across arXiv, Google Scholar, ACM DL, IEEE Xplore using specified keywords, plus snowballing from seminal papers identified by domain experts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is published literature, which is a standard corpus."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 and Section 2.2 document the full pipeline: Phase 1 (search + initial filtering → 311), Phase 2 (dedup + category assignment → 246), Phase 3 (full-text review with inclusion/exclusion criteria → 159)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Declarations section states funding: 'partially supported by National R&D Program of China (2024YFB4505603), Jiangsu Province Key R&D Program (Grant No. BG2024028) and National Natural Science Foundation of China (U23B2020, 62302479, 62232015).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: SKLP/ICT/CAS, UCAS, and University of Leeds."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Chinese national science foundations and R&D programs, which have no commercial stake in the survey's findings about LLM-compiler research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "Declarations section explicitly states: 'the corresponding author states that there is no Conflict of interest.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 follows a structured SLR protocol citing Kitchenham et al. (2007), with a three-phase search and selection process, explicit inclusion/exclusion criteria, and a flow diagram (Figure 1) showing paper counts at each stage."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its 159 source papers. All papers are treated equally regardless of rigor. Section 5.2 notes 'direct comparisons are not always perfectly fair' but does not systematically assess quality."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether its sources skew toward positive results or whether negative results in LLM-compiler research are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "159 primary studies were identified through a systematic three-phase literature search protocol.",
       "evidence": "Section 2.2 and Figure 1 describe the protocol: 311 candidates after Phase 1, 246 after dedup in Phase 2, 159 after full-text review in Phase 3.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs are integrated into compilation through three design philosophies: Selector, Translator, and Generator.",
       "evidence": "Sections 3.1-3.3 describe each philosophy with multiple examples from the literature. Selector: LLM chooses from predefined options. Translator: LLM directly rewrites code. Generator: LLM generates transformation scripts.",
       "supported": "strong"
     },
     {
       "claim": "There are more studies on high-level code transformations (59+34) than low-level IR/assembly (14+16).",
       "evidence": "Tables 2-4 enumerate the studies. The paper counts are derived from these tables and discussed in the text after Section 4.2.3.",
       "supported": "strong"
     },
     {
       "claim": "Neural compilation accuracy has progressed from 32.8% to over 99% on ExeBench.",
       "evidence": "Section 5.2 and Figure 6(b) track this progression: transformer-x86 (32.8%), llm-x86 (91.72%), LEGO-Compiler (99.0%).",
       "supported": "moderate"
     },
     {
       "claim": "The field lacks standardized comparison protocols, hindering fair direct evaluation.",
       "evidence": "Section 5.2 notes 'direct comparisons are not always perfectly fair' due to non-uniform setups. Section 6.2.3 discusses the need for standardized benchmarks. The GPU kernel comparison caveat notes 'different works may evaluate on different GPU hardware.'",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of source papers",
       "detail": "The survey treats all 159 papers equally without assessing their methodological quality. This means results from rigorous peer-reviewed studies are given the same weight as preliminary arXiv preprints, potentially laundering weak results into the survey's synthesis."
     },
     {
       "flag": "No publication bias discussion",
       "detail": "The survey does not consider whether its corpus skews toward positive results. LLM-compiler papers with negative findings (approaches that failed) are likely underrepresented in the literature."
     },
     {
       "flag": "No limitations of own methodology",
       "detail": "The paper discusses challenges in the field but does not discuss limitations of its own survey approach — e.g., potential selection bias from keyword choices, coverage gaps, or subjectivity in taxonomy assignment."
     },
     {
       "flag": "Uncritical reporting of SOTA evolution",
       "detail": "Section 5.2 tracks SOTA progress across tasks but acknowledges that 'direct comparisons are not always perfectly fair' without systematically addressing what this means for the claimed progress narrative. The progression figures may be misleading due to different evaluation setups."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and Codex, foundational to LLM code generation evaluation."
     },
     {
       "title": "Large Language Models for Compiler Optimization",
       "authors": ["Chris Cummins", "Volker Seeker"],
       "year": 2023,
       "arxiv_id": "2309.07062",
       "relevance": "Pioneered LLM pre-training on LLVM IR for compiler optimization, directly relevant to AI-assisted code optimization."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "First extensive study on applying LLMs directly for automated program repair."
     },
     {
       "title": "WhiteFox: White-box compiler fuzzing empowered by large language models",
       "authors": ["Chenyuan Yang"],
       "year": 2024,
       "relevance": "Multi-agent framework for LLM-based compiler fuzzing, relevant to agentic AI testing."
     },
     {
       "title": "KernelBench: Can LLMs write efficient GPU kernels?",
       "authors": ["Anne Ouyang"],
       "year": 2025,
       "relevance": "Key benchmark for evaluating LLM capabilities in GPU kernel generation and optimization."
     },
     {
       "title": "CUDA-L1: Improving CUDA optimization via contrastive reinforcement learning",
       "authors": ["Xiang Li"],
       "year": 2025,
       "relevance": "Demonstrates RL-based approach to CUDA optimization achieving near-perfect correctness, relevant to AI code optimization."
     },
     {
       "title": "Copiloting the copilots: Fusing large language models with completion engines for automated program repair",
       "authors": ["Yuxiang Wei", "Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "relevance": "LLM-based automated program repair combining completion engines, relevant to AI-assisted debugging."
     },
     {
       "title": "AlphaTrans: A neuro-symbolic compositional approach for repository-level code translation and validation",
       "authors": ["Ali Reza Ibrahimzada"],
       "year": 2025,
       "relevance": "Repository-level LLM-based code transpilation with verification, relevant to scalable AI code transformation."
     },
     {
       "title": "LEGO-Compiler: Enhancing Neural Compilation Through Translation Composability",
       "authors": ["Shuoming Zhang", "Jiacheng Zhao"],
       "year": 2025,
       "arxiv_id": "2505.20356",
       "relevance": "Achieved 99% neural compilation accuracy through divide-and-conquer, demonstrating LLM capability for end-to-end compilation."
     },
     {
       "title": "CoTran: An LLM-based code translator using reinforcement learning with feedback from compiler and symbolic execution",
       "authors": ["Prithwish Jana"],
       "year": 2024,
       "relevance": "RL-based feedback mechanism for LLM code translation, relevant to AI-assisted code migration."
     }
   ]
 }