ai-research-survey

scan.json (32346B)

---

 {
   "paper": {
     "title": "SysLLMatic: Large Language Models are Software System Optimizers",
     "authors": [
       "Huiyun Peng",
       "Arjun Gupte",
       "Ryan Hasler",
       "Nicholas John Eliopoulos",
       "Chien-Chou Ho",
       "Rishi Mantri",
       "Leo Deng",
       "Konstantin Läufer",
       "George K. Thiruvathukal",
       "James C. Davis"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.01249",
     "doi": "10.48550/arXiv.2506.01249"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SysLLMatic, an LLM-based code optimization system integrating a catalog of 43 performance patterns with profiling-guided feedback, achieves average 1.5× latency and 1.76× throughput improvements on five DaCapo real-world Java applications, outperforming compiler baselines on those. However, compiler baselines remain competitive or superior on microbenchmarks (HumanEval_CPP) and some DaCapo applications (Fop, ZXing). Open-source LLMs are competitive on microbenchmarks but fall short on large-scale applications. The BioJava parallelization result (3.44× latency, 4.63× throughput) dominates the DaCapo averages while other applications show modest or no improvement.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "§11 states 'Our artifact is: https://github.com/sysllmatic/sysllmatic.' A public URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All three benchmarks are publicly available: HumanEval-X, SciMark2, and DaCapo. The paper uses unmodified public benchmarks and provides an artifact repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "§6.7 describes hardware (Intel Xeon W-2295, 36 CPUs, 188 GB RAM; NVIDIA H100 GPUs) and mentions tools (ANTLR4, async-profiler, perf, JaCoCo, Lizard, ollama), but no requirements.txt, Dockerfile, or detailed dependency version listing is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the system architecture and methodology in detail (§5, Algorithm 1) but does not include step-by-step reproduction instructions. The prompt excerpts in §5.6.1 are explicitly shortened: 'These excerpts preserve the key instructions and style of the full prompts while omitting implementation-specific details.'"
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 8 and 9 report only point estimates (e.g., '1.55×', '88.41%'). Despite performing 5 measured runs per program (§6.4.1), no confidence intervals or error bars are reported on those averages."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SysLLMatic 'outperforms' baselines based solely on comparing point estimates in Tables 8 and 9. No statistical significance tests (t-tests, Wilcoxon, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative improvement ratios are consistently reported with baseline context throughout (e.g., '3.44× latency', '1.5× in latency vs. 1.01× for the compiler' in the abstract, Table 8). These provide clear effect size magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only 5 of 22 DaCapo applications were selected (§6.2.2) based on practical constraints (Maven build, test suite availability, file size limits), but no justification for this sample size being sufficient for general claims is provided. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "§6.4.1 states '2 warm-up runs followed by 5 measured runs for each program, reporting the average,' but no standard deviation, IQR, or any spread measure across those 5 runs is reported anywhere."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: PerfCodeGen (SOTA LLM-based optimizer), gcc -O3 compiler optimization, OpenTuner (for HumanEval_CPP), and JIT compiler flags (for Java). See §6.4.2 and Table 8."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "PerfCodeGen (2024) is described as SOTA at time of experiments (§6.4.2). Compiler baselines (gcc, JIT) are standard and actively maintained. The paper justifies excluding other large-scale LLM baselines due to incompatibility (§6.4.2)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "§6.5 and Table 11 present a thorough component-wise ablation (Base → +Evaluator → +Context → +Advisor) across all three benchmarks, plus system ablation varying feedback iterations (Figure 12) and hotspot budget K (Figure 14)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five performance metrics (latency, memory, CPU cycles, throughput, energy) plus correctness, maintainability metrics (CCN, function count, NLOC, tokens), and comparative metrics (Relative Improvement, %Opt). See Table 5."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "§7.1.4 describes manual inspection of all optimized outputs: 'To verify that performance gains were legitimate, we manually reviewed all outputs that passed automated tests.' §7.1.3 reports a manual qualitative analysis of 30 Advisor outputs with 100% hypothesis-recommendation alignment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The three benchmarks (HumanEval_CPP, SciMark2, DaCapo) serve as independent evaluation suites with their own test cases. No tuning was performed on the test sets; model hyperparameters (temperature 0.7) were fixed following prior work."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 provides per-application DaCapo results. Tables 8-9 break down results per benchmark. Figure 9 shows per-optimization-pattern distribution. Table 11 provides per-component ablation. Per-metric breakdowns throughout."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§7.1.4 discusses correctness failures: 1 HumanEval program had its core function removed but passed tests, 2 PMD files mismatched ground truth. §7.2.1 notes Fop and ZXing showed minimal improvement. §8 discusses scaling challenges and correctness limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "CPU regression in BioJava (0.48×, Table 8). Compiler outperforming SysLLMatic on Fop and ZXing. Open-source models degrading performance on DaCapo (DeepSeek-r1 at 0.90× latency, Table 9). Advisor causing regressions on microbenchmarks in ablation (Table 11). Llama4 at 0% SciMark2 correctness."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '1.5× in latency' and '1.76× in throughput' on DaCapo are supported by Table 8 (averaging BioJava 3.44×, Fop 1.02×, PMD 1.11×, Graphchi 0.99×, ZXing 0.95× for latency). The claim 'consistently outperforms state-of-the-art LLM baselines on microbenchmarks' is supported by Tables 8-9."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about component contributions (e.g., 'incorporating the optimization pattern catalog leads to improvements'). These are supported by controlled ablation studies (Table 11) with single-variable manipulation. The causal mechanism for BioJava's improvement (parallelization) is demonstrated with before/after code (Figure 8)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Large Language Models are Software System Optimizers' is a broad claim. The abstract states the work 'provides a foundation for generating optimized code across various languages, benchmarks, and program sizes.' However, evaluation covers only C++ and Java, 3 benchmarks, and 5 DaCapo apps. While §9 acknowledges generalization limitations, the title and framing significantly overshoot the tested scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§9 discusses specific alternative explanations: temperature-induced nondeterminism, hardware-level measurement variance. §8.1.5 discusses whether CPU-based profiling captures the right bottlenecks. §8.2.1 discusses test coverage limitations as alternative explanations for apparent correctness."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures the performance metrics it claims (latency, throughput, energy, etc.) through runtime execution. §6.3 precisely defines each metric and its measurement implementation. The gap between measurement and claim is small — they measure performance and claim performance improvement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "§6.6 specifies 'GPT-4o' and 'GPT-4.1' without snapshot dates or API versions. Open-source models are listed as 'qwen3-coder:480b, gemma3:27b, deepseek-r1:70b, and llama4:latest.' These are marketing names without specific version identifiers. Model behavior can change across API updates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "§5.6.1 provides 'shortened version of prompts' for Advisor, Generator, and Evaluator, explicitly stating 'These excerpts preserve the key instructions and style of the full prompts while omitting implementation-specific details.' The excerpts contain placeholders like '[Input items...]' and '[Output requirements...]'. Full prompts are not provided in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "§6.4.1: temperature 0.7, hotspot budget K=50, iteration budget T=2, pass@1, 2 warm-up + 5 measured runs. JIT compiler flags detailed in §6.4.2. Open-source models run via ollama (§6.6)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The Advisor→Generator→Evaluator pipeline is described in detail (§5, Figure 4, Algorithm 1). Profiling-based hotspot identification (§5.2), AST extraction, flame graph analysis, iterative feedback loops, and correctness validation are all documented. The workflow diagram and algorithm pseudocode make the scaffold reproducible."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§6.2.2 documents benchmark filtering: 22 DaCapo apps → 20 buildable → 8 with source+tests → 5 with Maven. Files over 1,000 lines skipped (§6.2.2). Classes without dedicated tests excluded (§5.6.3). HumanEval augmented with translated stress tests from COFFE benchmark (§6.2.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§8 Discussion dedicates multiple subsections to limitations (§8.1.1 progress and remaining challenges, §8.2 costs of applying SysLLMatic). §9 provides a dedicated Threats to Validity section with construct, internal, and external threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§9 discusses threats specific to this study: temperature 0.7 causing nondeterminism across runs, 5-run averaging for hardware variance mitigation, limited language coverage (C++ and Java only), DaCapo applications being pre-built, and incomplete training data for other languages. §8.2.1 discusses specific test coverage variations (GraphChi ~10% vs ZXing ~90%)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§9 External Threats: 'there are several aspects of generalizability that are not assessed, most notably to other programming languages, application types, and hardware platforms.' §8.1.1 explicitly states limitations: assumes access to test suites, DaCapo apps pre-built, profiling data time-consuming to collect."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw performance measurement data (individual run timings, profiling outputs, LLM outputs) is made available for independent verification. The paper reports only averaged/aggregated results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§6.7 describes the execution environment (Intel Xeon W-2295, 36 CPUs, 188 GB RAM). §6.3 defines each metric's measurement implementation (Linux /usr/bin/time for memory, perf stat for CPU cycles, Intel RAPL for energy). §6.4.1 specifies the measurement protocol (2 warm-up + 5 measured runs)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (HumanEval-X, SciMark2, DaCapo)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "§6.2.2 documents the DaCapo filtering pipeline with counts at each stage (22 → 20 → 8 → 5). §5 describes the full optimization pipeline from hotspot identification through code generation, validation, and profiling. The file size filtering criterion (>1,000 lines) and its impact (11% of Fop files excluded) are documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section: 'Davis acknowledges support from NSF awards #2343596. Thiruvathukal and Läufer acknowledge support from NSF award #2343595.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Purdue University (Peng, Gupte, Eliopoulos, Ho, Mantri, Deng, Davis) and Loyola University Chicago (Hasler, Läufer, Thiruvathukal). No evaluated product is affiliated with these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is a government funding agency with no financial stake in whether LLM-based code optimization outperforms compiler baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure section is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o, GPT-4.1, and several open-source models but never states training data cutoff dates for any of them. This is relevant because HumanEval (2021) and SciMark2 (2000) predate all models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and SciMark2 in 2000 — solutions could be in training data for all tested models. No discussion of potential train/test overlap anywhere in the paper."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and SciMark2 were available online well before any tested model's training cutoff. The paper uses these benchmarks without any discussion of contamination risk. This is particularly relevant since the LLM must understand the code's semantics to optimize it."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It evaluates LLM-based code optimization on software benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 12 reports end-to-end optimization time, number of LLM queries, and energy consumption (low/high bounds in Wh) per DaCapo application. §7.5.2 provides break-even analysis modeling inference costs against runtime and energy savings."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "§6.7 specifies hardware (Intel Xeon W-2295 bare-metal server, NVIDIA H100 GPU cluster). Table 12 reports time and energy per application (e.g., BioJava: 47 min, 54 queries, 13.0-324 Wh). The break-even analysis in §7.5.2 models total resource consumption."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0.7 (§6.4.1) introducing nondeterminism, and results use pass@1. No seed sensitivity analysis or results across multiple random seeds are reported. §9 acknowledges that 'identical prompts may yield different outputs across runs' but does not quantify this."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "§6.4.1: 'we perform 2 warm-up runs followed by 5 measured runs for each program, reporting the average of the measured runs.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is fixed at 0.7 'following prior work' (§6.4.1), K=50 is chosen based on covering 'all or most meaningful hotspots' (§5.6.2). While K is ablated across 3 values (50, 100, 150) in §7.4.2, no systematic hyperparameter search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Configuration choices are justified: temperature 0.7 from prior work convention (§6.4.1), K=50 from ablation showing diminishing returns (Figure 14), T=2 iterations from system ablation (Figure 12), class-level granularity from correctness comparison (§7.4.3, Figure 13)."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "§6.4.2 describes re-implementing PerfCodeGen for different languages: 'We reimplemented PerfCodeGen to support our benchmarks.' No acknowledgment of the Lucic et al. (2018) concern that authors' re-implementations of baselines systematically underperform."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 12 reports optimization cost (time, queries, energy) per application. §7.5 provides break-even analysis. The cost of LLM optimization is explicitly compared to gains. Figure 15 plots time-to-break-even as a function of execution frequency."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "§8.1.5 discusses whether CPU-based profiling captures the right performance dimensions, comparing hotspot profiles across event types (Figure 16). §6.2 justifies benchmark selection for different evaluation goals (synthetic vs. real-world). §8.1.3 discusses function-level vs. class-level validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Model comparisons in Table 9 use the same SysLLMatic framework across all models (GPT-4o, GPT-4.1, Qwen, Gemma, DeepSeek, Llama4). The PerfCodeGen comparison is explicitly a system-level comparison. §6.4.2 ensures 'same input code, configuration, environment, and evaluation metrics.'"
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021, SciMark2 in 2000 — solutions and benchmarks predate all tested models' training data. No discussion of whether models may have seen solutions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setup leaks answer information. The LLM receives source code and profiling data; it's not discussed whether optimization patterns for these specific benchmarks may have appeared in training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark programs share structural similarities with training data or with each other in ways that could inflate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used. No canary strings, membership inference tests, or decontamination pipelines."
       }
     }
   },
   "claims": [
     {
       "claim": "SysLLMatic achieves average 1.5× latency and 1.76× throughput improvement on DaCapoBench, surpassing compiler optimizations (1.01× and 1.02×).",
       "evidence": "Table 8 shows per-application results: BioJava 3.44×/4.63×, Fop 1.02×/1.07×, PMD 1.11×/1.12×, Graphchi 0.99×/0.98×, ZXing 0.95×/0.99×. Averages are heavily driven by BioJava's parallelization gains (§7.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "SysLLMatic consistently outperforms state-of-the-art LLM baselines (PerfCodeGen) on microbenchmarks.",
       "evidence": "Table 8: On SciMark2, SysLLMatic achieves 1.55× latency vs PerfCodeGen's 1.00×. On HumanEval, SysLLMatic achieves 88.41% correctness vs PerfCodeGen's 70%, and higher %Opt in latency (23.2% vs 15.0%).",
       "supported": "strong"
     },
     {
       "claim": "Compiler baselines remain highly competitive, particularly on HumanEval where they achieve higher %Opt than SysLLMatic in 3 of 4 metrics.",
       "evidence": "Table 8: Compiler achieves 61.0% CPU cycles %Opt vs SysLLMatic's 29.9% on HumanEval. Compiler outperforms on Fop (1.08× vs 1.02× latency) and ZXing (1.40× vs 0.95× latency) in DaCapo.",
       "supported": "strong"
     },
     {
       "claim": "The optimization pattern catalog is essential for large-scale DaCapo applications but not for microbenchmarks.",
       "evidence": "Table 11 ablation: Adding Advisor improves DaCapo latency from 1.07× to 1.50×, throughput from 1.06× to 1.76×. But on SciMark2, Advisor reduces latency from 6.25× to 1.55× compared to +Ctx configuration.",
       "supported": "strong"
     },
     {
       "claim": "Open-source LLMs achieve comparable performance on microbenchmarks but fall short on large-scale applications.",
       "evidence": "Table 9: Qwen3-Coder:480B achieves 93.9% correctness on HumanEval (vs GPT-4o's 88.4%), but only 1.03× latency on DaCapo (vs GPT-4.1's 1.50×). DeepSeek-r1 shows 0.90× latency regression on DaCapo.",
       "supported": "moderate"
     },
     {
       "claim": "BioJava achieves 3.44× latency and 4.63× throughput improvement through parallelization.",
       "evidence": "Table 8 and Figure 8 show before/after code with thread pool parallelization. However, CPU cycles regress to 0.48×, reflecting concurrency overhead (§7.2.1).",
       "supported": "strong"
     },
     {
       "claim": "SysLLMatic's optimization cost is worthwhile for high-throughput applications, with BioJava breaking even within 5-8 days under conservative usage assumptions.",
       "evidence": "§7.5.2 models break-even at 230 executions (energy) and 414 executions (latency) for BioJava, translating to 5 days and 8 days at 50 executions/day. PMD requires months to years (Figure 15).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "Despite performing 5 measured runs per program (§6.4.1), no standard deviation, confidence intervals, or error bars are reported. With temperature 0.7 introducing nondeterminism in LLM outputs, the stability of results is unquantified. The paper acknowledges this threat (§9) but does not address it."
     },
     {
       "flag": "BioJava dominates DaCapo averages",
       "detail": "The headline '1.5× latency, 1.76× throughput' on DaCapo is driven almost entirely by BioJava (3.44×, 4.63×). The other 4 applications average ~1.02× latency and ~1.04× throughput. Reporting the average without emphasizing this skew is misleading."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "HumanEval (2021) and SciMark2 (2000) predate all tested models' training. GPT-4o/4.1 and open-source models may have seen optimization solutions for these benchmarks. No discussion of contamination risk despite this being a well-known concern."
     },
     {
       "flag": "Baseline re-implementation",
       "detail": "§6.4.2: PerfCodeGen was re-implemented by the authors for different languages. Lucic et al. (2018) showed that authors' re-implementations of baselines systematically underperform. The original PerfCodeGen authors were not involved in verification."
     },
     {
       "flag": "Very small DaCapo evaluation set",
       "detail": "Only 5 of 22 DaCapo applications were used, selected for convenience (Maven build system, available test suites). The filtering is honest but the resulting sample is small and may not represent the diversity of real-world software systems."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "GPT-4o and GPT-4.1 are used without snapshot dates or API versions. Model behavior changes across versions; results may not be reproducible with future API updates."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models for code optimization: Survey, challenges and future directions",
       "authors": ["J. Gong", "V. Voskanyan", "P. Brookes"],
       "year": 2025,
       "arxiv_id": "2501.01277",
       "relevance": "Major survey of LLM-based code optimization cataloging limitations of current approaches, directly motivating SysLLMatic's design."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark used as one of three evaluation suites in this paper."
     },
     {
       "title": "EffiLearner: Enhancing efficiency of generated code via self-optimization",
       "authors": ["D. Huang", "J. Dai", "H. Weng"],
       "year": 2024,
       "arxiv_id": "2405.15189",
       "relevance": "Self-optimization pipeline using overhead profiles for code efficiency, representative of feedback-based LLM optimization approaches."
     },
     {
       "title": "PerfCodeGen: Improving performance of LLM generated code with execution feedback",
       "authors": ["Y. Peng", "A. D. Gotmare", "M. Lyu"],
       "year": 2024,
       "arxiv_id": "2412.03578",
       "relevance": "Primary LLM-based baseline, SOTA training-free framework using unit test runtime to guide performance improvements."
     },
     {
       "title": "DeepDev-PERF: a deep learning-based approach for improving software performance",
       "authors": ["S. Garg", "R. Z. Moghaddam", "Clement"],
       "year": 2022,
       "relevance": "Fine-tuned BART for real-world C# performance optimization; one of few prior works targeting real-world applications."
     },
     {
       "title": "Search-Based LLMs for Code Optimization",
       "authors": ["S. Gao", "C. Gao", "Gu"],
       "year": 2025,
       "relevance": "Search-based approach to LLM code optimization, part of the growing literature on automated code performance improvement."
     },
     {
       "title": "MARCO: A multi-agent system for optimizing HPC code generation using large language models",
       "authors": ["A. Rahman", "V. Cvetkovic", "K. Reece"],
       "year": 2025,
       "arxiv_id": "2505.03906",
       "relevance": "Multi-agent framework for HPC code optimization, representing the agent-based approach to code optimization."
     },
     {
       "title": "Learning performance-improving code edits",
       "authors": ["A. G. Shypula", "A. Madaan", "Y. Zeng"],
       "year": 2024,
       "relevance": "Foundational work on LLM-based performance-improving code edits, establishes conventions used as baselines."
     },
     {
       "title": "When Faster Isn't Greener: The Hidden Costs of LLM-Based Code Optimization",
       "authors": ["T. Coignion", "C. Quinton", "R. Rouvoy"],
       "year": 2025,
       "relevance": "Examines sustainability costs of LLM-based optimization, directly motivating SysLLMatic's break-even analysis."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs for software engineering tasks providing context for LLM code optimization capabilities."
     },
     {
       "title": "Large Language Models for energy-efficient code: Emerging results and future directions",
       "authors": ["H. Peng", "A. Gupte", "N. J. Eliopoulos"],
       "year": 2024,
       "arxiv_id": "2410.09241",
       "relevance": "Prior work by some of the same authors on LLM-guided energy-efficient code optimization through prompting."
     },
     {
       "title": "RAPGen: An approach for fixing code inefficiencies in zero-shot",
       "authors": ["S. Garg", "R. Z. Moghaddam", "N. Sundaresan"],
       "year": 2025,
       "arxiv_id": "2306.17077",
       "relevance": "Retrieval-augmented prompting for code performance bug repair in real-world projects, one of few prior works on application-level optimization."
     }
   ]
 }