ai-research-survey

scan.json (26472B)

---

 {
   "paper": {
     "title": "Language Models for Code Optimization: Survey, Challenges and Future Directions",
     "authors": [
       "Jingzhi Gong",
       "Vardan Voskanyan",
       "Paul Brookes",
       "Fan Wu",
       "Wei Jie",
       "Jie Xu",
       "Rafail Giavrimis",
       "Mike Basios",
       "Leslie Kanthan",
       "Zheng Wang"
     ],
     "year": 2025,
     "venue": "ACM Computing Surveys (arXiv preprint)",
     "arxiv_id": "2501.01277",
     "doi": "10.48550/arXiv.2501.01277"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This SLR of 53 primary studies finds that general-purpose LMs (61 instances) were more widely adopted for code optimization than code-specialized LMs (43 instances), with 57% of studies using off-the-shelf models. 81% of studies targeted a single programming language and 79% a single performance metric, highlighting limited generalizability. Only 32% of studies evaluated on real-world code, with 68% relying on competitive programming or synthetic datasets. Five open challenges are identified including balancing model complexity with practicality, limited interaction with external systems, and trust/reliability in AI-driven optimization.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Full list of studies and all the raw results of this survey can be accessed at: https://github.com/gjz78910/CodeOpt-SLR' (Section 1, footnote 1). A repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository is stated to contain 'all the raw results of this survey,' and the full methodology and related works are also hosted there. The extracted data from the 53 primary studies appears to be released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or setup instructions are mentioned in the paper for reproducing the survey analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The methodology is described at a high level in Section 3, but specific steps to reproduce the analysis are not given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that counts and categorizes studies. It does not perform statistical aggregation or meta-analysis requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The survey reports descriptive statistics (counts, percentages) of categorized studies. No comparative statistical claims requiring significance tests are made."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No effect sizes are relevant for a categorization-based survey without statistical aggregation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper identifies 53 primary studies from 2,346 searched studies but does not justify whether 53 is sufficient for the breadth of conclusions drawn, nor discuss whether the search may have missed relevant studies beyond acknowledging the quasi-gold standard approach."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs or statistical aggregation requiring variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that existing reviews focus on general SE applications or program repair (Section 1) but does not formally compare its scope, methodology, or findings against those prior surveys."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Prior surveys are referenced but not systematically compared. The paper claims to fill a gap but does not benchmark its coverage or findings against specific prior reviews."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate; this is a survey paper."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; this is a survey paper."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No experimental outputs to evaluate; this is a survey paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with train/test splits; this is a survey paper."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides detailed breakdowns across multiple dimensions: LM types (Table 1), model sizes (Figure 6), training approaches (Figure 7), challenges (Table 2), optimization techniques (Table 3), LM roles (Table 4), programming languages (Table 5), performance metrics (Table 6), datasets (Table 7), and evaluation approaches (Figures 10-11)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses five open challenges including limited generalizability, lack of real-world evaluation, hallucination issues, and trust/reliability concerns. The paper also notes limitations of each technique category throughout Sections 4-7."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: 68% of studies were not evaluated on real-world code (Section 7.2), 81% focused on single languages (Section 6.1), and LMs 'currently struggle with larger programs and often yield marginal improvements over traditional compilers' (Section 2.3, citing reference [113])."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about general-purpose LMs (61 instances), 57% pre-trained, 43% fine-tuned, and the specific challenge counts are all supported by the corresponding tables and figures in the body (Tables 1-3, Figure 7)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'General-purpose LMs like GPT-4 were more widely adopted... due to their broader understanding and reasoning capabilities' (Section 4.1) without evidence for the causal mechanism. The 'due to' attributions for adoption patterns are unsubstantiated interpretations rather than findings from the survey data."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The scope is clearly bounded to 'LM-based approaches for code optimization' (Section 1), and Figure 1 explicitly shows the survey scope. The title accurately reflects the content, and conclusions are generally tied to the surveyed literature."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents observed patterns (e.g., dominance of general-purpose LMs, preference for single-language studies) with single explanations and does not consider alternative reasons. For instance, the popularity of GPT-4 could be due to API accessibility rather than 'broader understanding,' but alternatives are not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims are at the level of its measurements — counting and categorizing studies. It does not frame study counts as proxies for broader phenomena. When it draws conclusions about the field, these are stated as observations from the surveyed literature."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models were used in conducting this survey."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting was used in this survey."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models or hyperparameters involved in conducting this survey."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used in this survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the three-stage methodology: (1) search using six academic indexing engines with a defined search string and snowballing, (2) study selection using inclusion/exclusion criteria and quality assessment, (3) data collection guided by 4 RQs with 11 sub-questions. Figure 4 shows the pipeline with counts (2,346 → 53). However, full criteria details are deferred to the GitHub repo."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section for the survey methodology itself. Section 8 discusses challenges in the field (limitations of LM-based code optimization), not limitations of this survey. The conclusion briefly notes 'it is impossible to provide a definitive cataloger of all research' but this is a single sentence, not substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to the validity of the survey methodology are discussed. There is no mention of potential biases in the search process, coding reliability, or limitations of the categorization approach."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes code optimization from code generation, refactoring, and repair (Section 2.1). Figure 1 visually delineates the survey scope. The definition of code optimization is clearly stated as 'techniques that enhance performance objectives while preserving the original functionality.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository (https://github.com/gjz78910/CodeOpt-SLR) is stated to contain 'all the raw results of this survey,' enabling independent verification of the categorizations."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection procedure: comprehensive automatic searches through six academic indexing engines using a defined search string following the quasi-gold standard methodology, supplemented by snowballing searches. The three-stage process is outlined in Figure 4."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper selection process is described under data_collection_described and data_preprocessing_documented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 documents the pipeline: manual search (10 studies for quasi-gold standard) + automatic search (2,310 studies) + snowballing → 2,346 total → inclusion/exclusion criteria → quality assessment → 53 primary studies → data collection and analysis. Counts at each stage are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed. Multiple authors (Voskanyan, Brookes, Wu, Giavrimis, Basios, Kanthan) are affiliated with TurinTech AI, a company that works on AI-based code optimization. University affiliations are also listed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. With six authors from TurinTech AI (a company commercializing AI for code optimization), the lack of any funding or conflict disclosure is a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Six authors are from TurinTech AI, which has a commercial interest in the field being surveyed, but this potential conflict is not acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method of its own."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The paper follows the Kitchenham and Charters (2007) guidelines for SLRs in Software Engineering (Section 3). It uses a quasi-gold standard methodology for search string construction, applies structured inclusion/exclusion criteria, and documents the pipeline in Figure 4 with a flow diagram showing counts at each stage."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "Quality assessment is mentioned as part of the selection pipeline (Figure 4) for inclusion/exclusion purposes, but the survey does not assess or report the methodological quality of the 53 included studies. All studies are treated equally in the analysis regardless of their rigor. No quality scores, risk-of-bias ratings, or strength-of-evidence assessments are applied."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether the 53 included studies skew toward positive results, does not use funnel plots or tests for publication bias, and does not acknowledge that published LM-for-code-optimization papers may disproportionately report successes."
       }
     }
   },
   "claims": [
     {
       "claim": "General-purpose LMs were more widely adopted (61 instances) than code-specialized LMs (43 instances) for code optimization, with GPT-4 being the most frequently used (15 studies).",
       "evidence": "Table 1 in Section 4.1 provides a detailed breakdown of all LMs used across the 53 primary studies, listing each model's usage count with study references.",
       "supported": "strong"
     },
     {
       "claim": "57% of studies leveraged off-the-shelf pre-trained LMs while 43% fine-tuned models for specific tasks.",
       "evidence": "Figure 7 in Section 4.3 shows the distribution, with the underlying study references provided for each category.",
       "supported": "strong"
     },
     {
       "claim": "The most commonly highlighted challenges were limitation of one-step optimization (18 studies), balancing correctness and efficiency (15 studies), and complexity of code syntax (10 studies).",
       "evidence": "Table 2 in Section 5.1 provides the full categorization of challenges with specific study references for each.",
       "supported": "strong"
     },
     {
       "claim": "81% of studies focused on a single programming language and 79% targeted a single performance metric.",
       "evidence": "Figures 8 and 9 in Section 6 show the distributions, with 43/53 studies on one language and 42/53 on one metric.",
       "supported": "strong"
     },
     {
       "claim": "68% of studies (36 of 53) were not evaluated on real-world code projects, relying instead on competitive programming or synthetic datasets.",
       "evidence": "Figure 10 and Section 7.2 break down evaluation into no real-world (36), code snippets (12), and full projects (5).",
       "supported": "strong"
     },
     {
       "claim": "Model-based techniques were the most popular approach (51 instances), followed by prompt engineering (34 instances) and problem formulation (33 instances).",
       "evidence": "Table 3 in Section 5.2 provides the full categorization with study references for each technique and sub-category.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Six of ten authors are affiliated with TurinTech AI, a company that commercializes AI-based code optimization tools (their product Artemis++ is cited as reference [43]). No conflict of interest statement, competing interests declaration, or funding disclosure is present. A survey of the field by employees of a company operating in that field warrants explicit disclosure."
     },
     {
       "flag": "No quality assessment of surveyed studies",
       "detail": "The survey treats all 53 included studies equally without assessing their methodological quality, risk of bias, or strength of evidence. This risks laundering weak or poorly-designed studies alongside rigorous ones, making the aggregate counts and percentages potentially misleading."
     },
     {
       "flag": "Publication bias not addressed",
       "detail": "The survey does not discuss whether the included studies skew toward positive results. Published LM-for-code-optimization papers likely overrepresent successes, but this systematic bias is not acknowledged or mitigated."
     },
     {
       "flag": "Key methodology details deferred to external repository",
       "detail": "The full search methodology, related works, and inclusion/exclusion criteria details are not in the paper but deferred to the GitHub repo ('Due to space limitation, the full methodology can be accessed in our repository'). This makes independent assessment of the survey's rigor dependent on external resources."
     },
     {
       "flag": "No limitations section for the survey itself",
       "detail": "While Section 8 discusses challenges in the field, there is no discussion of limitations of the survey methodology — no threats to validity, no inter-coder reliability assessment, no discussion of potential gaps in the search strategy."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Key LLM self-improvement technique applied to code optimization, demonstrating iterative feedback mechanisms for code reasoning."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2023,
       "relevance": "Agentic LLM framework using verbal reinforcement learning for code optimization and decision-making tasks."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Most widely used code-specialized LM in the surveyed studies (11 instances), foundational model for code optimization research."
     },
     {
       "title": "Large Language Models for Compiler Optimization",
       "authors": ["Chris Cummins", "Volker Seeker", "Dejan Grubisic"],
       "year": 2023,
       "arxiv_id": "2309.07062",
       "relevance": "Pioneering work on LLMs as compiler emulators for code optimization at the IR level."
     },
     {
       "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
       "authors": ["Junwei Liu", "Kaixin Wang", "Yixuan Chen"],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Comprehensive survey of LLM-based agents for software engineering tasks including code optimization."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2023,
       "relevance": "Prior SLR on LLMs in software engineering, positioned as complementary to this code-optimization-specific survey."
     },
     {
       "title": "A Systematic Literature Review on Large Language Models for Automated Program Repair",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Yang Xie"],
       "year": 2024,
       "arxiv_id": "2405.01466",
       "relevance": "Related SLR focusing on LLMs for automated program repair, adjacent to code optimization."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and Codex, foundational benchmarks and models used across the surveyed code optimization studies."
     },
     {
       "title": "Learning Performance-Improving Code Edits",
       "authors": ["Alexander Shypula", "Aman Madaan", "Yimeng Zeng"],
       "year": 2024,
       "relevance": "Introduces the PIE dataset with 77K slow-fast code pairs, a foundational dataset for LM-based code performance optimization."
     },
     {
       "title": "Mathematical Discoveries from Program Search with Large Language Models",
       "authors": ["Bernardino Romera-Paredes", "Mohammadamin Barekatain", "Alexander Novikov"],
       "year": 2024,
       "relevance": "FunSearch: demonstrates LLMs searching program space for optimization, achieving mathematical discoveries through code."
     },
     {
       "title": "Search-Based LLMs for Code Optimization",
       "authors": ["Shuzheng Gao", "Cuiyun Gao", "Wenchao Gu", "Michael Lyu"],
       "year": 2024,
       "relevance": "SBLLM framework combining evolutionary search with LLMs for code optimization, addressing the search-based approach category."
     },
     {
       "title": "DeepDev-PERF: A Deep Learning-Based Approach for Improving Software Performance",
       "authors": ["Spandan Garg", "Roshanak Zilouchian Moghaddam", "Colin B. Clement"],
       "year": 2022,
       "relevance": "Deep learning approach for real-world C# performance optimization using GitHub data, one of few studies evaluating on full projects."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "As a survey, it provides useful reference tables and taxonomy for researchers but is not directly actionable for practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that 68% of studies lack real-world evaluation and 81% target single languages is mildly surprising but not deeply contrarian."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised; the paper focuses on code performance optimization."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or dramatic claims; straightforward survey with moderate conclusions."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No tool, demo, or runnable artifact; the GitHub repo contains survey data only."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Discusses well-known models (GPT-4, Copilot, Code Llama) but the authoring institutions are not major AI labs."
     }
   }
 }