ai-research-survey

scan.json (29842B)

---

 {
   "paper": {
     "title": "GREEN-CODE: Learning to Optimize Energy Efficiency in LLM-based Code Generation",
     "authors": [
       "Shashikant Ilager",
       "Lukas Florian Briem",
       "Ivona Brandić"
     ],
     "year": 2025,
     "venue": "IEEE/ACM International Symposium on Cluster, Cloud and Internet Computing",
     "arxiv_id": "2501.11006",
     "doi": "10.1109/ccgrid64434.2025.00068"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GREEN-CODE uses RL-based dynamic early exiting to reduce LLM inference energy consumption by 23-50% for code generation tasks. Fine-tuning with weighted aggregated loss enables intermediate layer decoding with a single LM head, avoiding the overhead of multiple LM heads. Evaluated on Llama 3.2 3B and OPT 2.7B with JavaCorpus and PY150, the approach achieves configurable accuracy-efficiency trade-offs controlled by a softmax temperature threshold.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in Section V: https://github.com/Large-scale-Sustainable-Computing-LSC/green-code, described as containing 'prototype implementation, deployment configuration scripts, and details, including libraries and dependencies used.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available JavaCorpus and PY150 datasets from the CodeXGlue benchmark. Table I cites the original sources and Section III-B describes following CodeXGlue's train-test splits."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (NVIDIA RTX 8000, AMD EPYC 7452, Ubuntu 20.04.6 LTS, Python 3.12) and libraries are named (Gymnasium, Stable-Baselines3, PyTorch, ZeusMonitor), but no library versions are specified in the paper. The GitHub repo is said to contain dependency details, but the paper itself lacks a requirements file or version-level environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper points to a GitHub repository but does not include step-by-step reproduction instructions within the paper itself. No 'Reproducing Results' section or specific commands are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Figures 8-13 are presented as point estimates (bar charts) without error bars, confidence intervals, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims such as '23-50% energy reduction' and performance comparisons between GC variants and baselines are made without any statistical significance tests. No p-values, t-tests, or other statistical tests appear anywhere in the paper."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes with baseline context are reported throughout: e.g., 'RougeL score of about 0.41, while the full model achieves around 0.425, while saving 23% in energy and time' (Section VI-E1). Percentage reductions and absolute score comparisons are consistently provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We always evaluate on 1000 samples from the test sets' (Section VI-C) but provides no justification for this number and no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results appear to be from single runs. No mention of multiple experimental runs or result stability across repetitions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are compared: (i) the base (non-fine-tuned) model using all layers, and (ii) the fine-tuned model using all layers without early exits. Results are compared across all metrics in Figures 8-11."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are the authors' own base and fine-tuned models. No comparison against contemporary competing early-exit methods such as LayerSkip (2024), ConsistentEE (2024), or Sun et al. (2024), all of which are discussed in related work (Section VII) as directly relevant approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison between base model (no fine-tuning, no RL), full model (fine-tuned, all layers), and GC variants (fine-tuned + RL at various thresholds) shows the contribution of the fine-tuning and RL agent components. The sensitivity analysis of context length (Section VI-F) and KV cache impact (Section VI-G) provide additional ablation-like analyses."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: RougeL, BLEU, CodeBLEU (with syntax and dataflow sub-metrics), energy consumption (Ws), latency (seconds), throughput (tokens/second), and number of layers skipped."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using text-matching and code-specific metrics. No human evaluation of generated code quality was conducted, despite the paper acknowledging that 'Standard LLM metrics like BLEU and ROUGE...may not align with the evaluation of the functional aspects of the code.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-C states: 'We follow the train-test split introduced in CodeXGlue.' The RL agent is trained on the training set, and evaluation uses 1000 samples from the test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (Llama, OPT), per dataset (JavaCorpus, PY150), per threshold (T = 0.6 to 0.92), and per context length (0.2 to 0.6) in Figures 8-12."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative error analysis or specific failure examples are shown. The paper notes performance degradation at high context settings (Section VI-F) and KV cache limitations (Section VI-G) but does not examine what types of code completions the early-exit approach gets wrong."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings reported: fine-tuning slightly reduces accuracy vs base model on most metrics (Section VI-E1), performance gap widens at higher context lengths ('almost double the loss observed,' Section VI-F), and KV cache is natively incompatible with early exits (Section VI-G). OPT is noted to perform worse than Llama."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'reduces the energy consumption between 23-50% on average...without significantly affecting accuracy.' Results in Section VI-E show 23% savings at T=0.92 for Llama/JavaCorpus and ~40-50% at lower thresholds, with accuracy trade-offs. The claim is supported, though 'without significantly affecting accuracy' applies mainly to the highest threshold settings."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims the RL-based early exit causes energy reduction. This is justified through controlled manipulation: comparing the same model with and without the RL agent component, isolating the effect of early exiting. The design allows for valid causal attribution of energy savings to the early exit mechanism."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Learning to Optimize Energy Efficiency in LLM-based Code Generation' implies broad applicability, but the paper tests only two small models (2.7B, 3B parameters) on two datasets. The conclusion acknowledges plans to 'extend this framework to incorporate larger models for different LLM tasks' but the title and abstract do not bound claims to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider whether energy savings could be attributed to other factors (e.g., reduced batch processing, GPU power management effects), or whether the RL agent's behavior might be task-specific rather than generalizable."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A2 explicitly acknowledges the proxy gap: 'Standard LLM metrics like BLEU and ROUGE, provide a broad performance estimate, prioritizing lexical precision, which may not align with the evaluation of the functional aspects of the code.' They add CodeBLEU with syntax and dataflow sub-metrics to partially address this gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific models are identified: 'Llama 3.2' with 3B parameters and 28 layers, and 'OPT' with 2.7B parameters and 32 layers (Table II). These are specific open-source model releases with fixed weights on Huggingface."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. The models perform autoregressive code completion given code context, using fine-tuning and RL-based inference rather than prompt-based interaction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table III reports PPO hyperparameters (steps, batch size, buffer size, epochs, learning rate, discount factor, network architecture). Section III-D reports fine-tuning hyperparameters (learning rate 1e-5, batch size 4, gradient accumulation 32, number of epochs, weight budget α values)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a model-level inference optimization (early exiting), not an agent pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B describes dataset selection from CodeXGlue, train-test splits, and preprocessing including tokenization and normalization. Section VI-C describes context selection (first 20% of tokens), maximum context length (512 tokens), and maximum new tokens (15)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. The conclusion mentions future plans ('we plan to extend this framework to incorporate larger models for different LLM tasks') but does not substantively discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no consideration of internal or external validity threats specific to the study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not identify what the results do NOT show, what model sizes or tasks are excluded, or what claims the authors are NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmark datasets are public and code is on GitHub, the raw experimental outputs (energy measurements, model predictions, per-sample results) are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are described: JavaCorpus and PY150 from CodeXGlue (Section III-B), with sample counts and token statistics (Table I). Energy is measured using ZeusMonitor/pynvml. Evaluation uses 1000 test samples with specific context settings."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (JavaCorpus, PY150)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented across Sections III-IV: dataset selection → preprocessing (tokenization, splitting, sequence length limits) → fine-tuning with aggregated loss → RL agent training → evaluation. Context selection and evaluation procedures are specified in Section VI-C."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists three funding sources: HPQC (Austrian Research Promotion Agency FFG Nr. 897481), Triton (Austrian Science Fund FWF DOI: 10.55776/P36870), and Themis (FWF DOI: 10.55776/PAT1668223)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Amsterdam and TU Wien. No product of a specific company is being evaluated, so no conflict arises."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Austrian public research agencies (FFG and FWF), which are government bodies with no financial stake in the outcome of the research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for either Llama 3.2 or OPT. The paper does not discuss when the training data for these models was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. JavaCorpus (2013) and PY150 (2016) were published well before Llama 3.2 (2024) and OPT (2022), meaning these benchmark solutions could be in the models' training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Both JavaCorpus (2013) and PY150 (2016) were available online years before the training of Llama 3.2 and OPT. The paper does not discuss this contamination risk. While contamination affects absolute scores less than relative comparisons, it is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference cost is a central focus. Energy consumption (Ws), latency (seconds), and throughput (tokens/second) are reported for all configurations in Figures 8-11. Table IV reports overhead percentages."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware specified (NVIDIA RTX 8000, AMD EPYC 7452). Fine-tuning time reported: ~24 hours for Llama, ~19 hours for OPT (Section III-D). RL training took 200k-500k steps to converge (Section V)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from single runs with no seed sensitivity analysis, despite RL training being known to be sensitive to random initialization."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented without indicating whether they come from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly states: 'we did not perform hyperparameter optimization due to the significant computational costs associated with it' (Section VI-B). Default library parameters were used."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Rather than selecting a single 'best' configuration, the paper transparently presents results across multiple thresholds (T = 0.6, 0.8, 0.9, 0.91, 0.92) and lets readers choose based on their own accuracy-efficiency trade-off preferences. For fine-tuning, default hyperparameters are acknowledged."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied. The paper makes numerous comparative claims (across thresholds, models, datasets) purely from point estimates."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their GREEN-CODE system against their own implementations of baselines (base model and fine-tuned model) without acknowledging potential self-comparison bias. No independent evaluation or external baseline implementations are used."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "This is a central contribution of the paper. Performance is systematically reported as a function of compute/energy at each threshold level, and Table IV reports overhead explicitly relative to the full model."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A2 discusses the limitations of BLEU and ROUGE for code evaluation and adds CodeBLEU with syntax and dataflow sub-metrics: 'code generation demands additional syntactic and semantic correctness to ensure functionality. Standard LLM metrics like BLEU and ROUGE...may not align with the evaluation of the functional aspects of the code.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The system performs model-level inference optimization (early exiting), not agent-based code generation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "JavaCorpus (2013) and PY150 (2016) were published years before Llama 3.2 (2024) and OPT (2022) were trained. The models could have seen the benchmark solutions during pre-training. This temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The context is drawn from the same code files as the ground truth, which is standard practice but not discussed as a potential concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data. The paper uses CodeXGlue splits but does not verify whether pre-trained model training data overlaps with test examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "GREEN-CODE reduces energy consumption between 23-50% on average for code generation tasks without significantly affecting accuracy.",
       "evidence": "Figures 8-11 show energy savings across models and datasets. For Llama/JavaCorpus at T=0.92, RougeL drops from ~0.425 to ~0.41 (3.5% drop) with 23% energy savings. At T=0.6, ~50% energy savings but RougeL drops to ~0.29 (~31% drop). Similar patterns for OPT and PY150.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning with aggregated loss enables intermediate layer decoding with a single LM head, eliminating the need for multiple LM heads.",
       "evidence": "Section III-D describes the LITE-based fine-tuning method. Figure 4 shows loss convergence. Figure 1 demonstrates that fine-tuned models achieve reasonable CodeBLEU even at shallow layers (e.g., 0.3 at layer 10 vs 0.4 at final layer for Llama).",
       "supported": "strong"
     },
     {
       "claim": "The RL agent learns to balance trade-offs between accuracy, latency, and energy consumption through dynamic early exiting.",
       "evidence": "Figure 6 shows PPO training convergence across 2000-2500 episodes. Figure 7 shows distribution of optimal exit points. Section VI-E shows the agent achieves different accuracy-efficiency trade-offs at different thresholds.",
       "supported": "moderate"
     },
     {
       "claim": "The overhead of the RL agent is reasonable — always below 1/5th of the total runtime.",
       "evidence": "Table IV reports energy and time overhead for different thresholds. Maximum overhead is 19.43%/11.98% for OPT at T=0.92. The overhead scales with threshold since higher thresholds cause more 'continue' actions.",
       "supported": "strong"
     },
     {
       "claim": "GREEN-CODE achieves comparable accuracy to the non-fine-tuned base model using all layers.",
       "evidence": "Figures 8-11 show that at high thresholds (T=0.91-0.92), GC approaches base model accuracy. However, at T=0.92 for Llama/JavaCorpus, RougeL is ~0.41 vs ~0.425 for the base model (still a gap), and at lower thresholds the gap widens substantially.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All experiments appear to be single-run results with no variance, standard deviation, or confidence intervals. RL training is known to be highly sensitive to random seeds (Henderson et al. 2018 found results can vary by 2x), making single-run PPO results unreliable."
     },
     {
       "flag": "No comparison with competing methods",
       "detail": "Despite discussing LayerSkip (2024), ConsistentEE (2024), and Sun et al. (2024) in related work as directly relevant early-exit approaches for LLMs and code generation, the paper does not compare against any of them experimentally."
     },
     {
       "flag": "Overstated abstract claim",
       "detail": "The abstract claims energy reduction 'without significantly affecting accuracy,' but at aggressive settings (T=0.6), RougeL drops ~31% and CodeBLEU drops ~43%. Even at the most conservative setting (T=0.92), accuracy is still measurably lower. The claim applies only to the narrowest interpretation."
     },
     {
       "flag": "Limited model scale",
       "detail": "Only 2.7B and 3B parameter models are tested, while production code generation tools use models orders of magnitude larger. The paper's title suggests general applicability to 'LLM-based Code Generation' but results may not transfer to larger models where layer dynamics differ."
     },
     {
       "flag": "No statistical testing",
       "detail": "Numerous comparative claims are made (across thresholds, models, datasets) without any statistical significance testing. Differences are presented as meaningful based purely on point estimate comparisons."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating the code quality of AI-assisted code generation tools: An empirical study on GitHub Copilot, Amazon CodeWhisperer, and ChatGPT",
       "authors": ["B. Yetistiren", "I. Özsoy", "M. Ayerdem", "E. Tüzün"],
       "year": 2023,
       "arxiv_id": "2304.10778",
       "relevance": "Empirical evaluation of AI code generation tool quality, directly relevant to the survey's coverage of LLM code generation assessment."
     },
     {
       "title": "CodeXGlue: A machine learning benchmark dataset for code understanding and generation",
       "authors": ["S. Lu", "D. Guo", "S. Ren"],
       "year": 2021,
       "relevance": "Major code understanding/generation benchmark suite used in this paper; central to evaluating LLM code capabilities."
     },
     {
       "title": "When neural code completion models size up the situation: Attaining cheaper and faster completion through dynamic model inference",
       "authors": ["Z. Sun", "X. Du", "F. Song", "S. Wang", "L. Li"],
       "year": 2024,
       "relevance": "Most closely related work on early exiting for code generation with classifier-based approach and multiple LM heads."
     },
     {
       "title": "ConsistentEE: A consistent and hardness-guided early exiting method for accelerating language models inference",
       "authors": ["Z. Zeng", "Y. Hong", "H. Dai", "H. Zhuang", "C. Chen"],
       "year": 2024,
       "relevance": "RL-based early exiting method with policies and LM heads at every exit layer, directly comparable approach to GREEN-CODE."
     },
     {
       "title": "LayerSkip: Enabling early exit inference and self-speculative decoding",
       "authors": ["M. Elhoushi", "A. Shrivastava", "D. Liskovich"],
       "year": 2024,
       "relevance": "Contemporary early-exit method using dropout during training with self-speculative decoding verification, relevant to LLM inference efficiency."
     },
     {
       "title": "LLM-pruner: On the structural pruning of large language models",
       "authors": ["X. Ma", "G. Fang", "X. Wang"],
       "year": 2023,
       "relevance": "Structural pruning approach for LLMs, alternative technique for reducing LLM inference cost relevant to the survey."
     },
     {
       "title": "QLoRA: Efficient finetuning of quantized LLMs",
       "authors": ["T. Dettmers", "A. Pagnoni", "A. Holtzman", "L. Zettlemoyer"],
       "year": 2024,
       "relevance": "Efficient LLM fine-tuning via quantization, relevant to resource-efficient LLM deployment approaches."
     },
     {
       "title": "CodeBLEU: A method for automatic evaluation of code synthesis",
       "authors": ["S. Ren", "D. Guo", "S. Lu"],
       "year": 2020,
       "relevance": "Code-specific evaluation metric incorporating syntax and dataflow matching, key to assessing LLM code generation quality."
     },
     {
       "title": "Out of the BLEU: How should we assess quality of the code generation models?",
       "authors": ["M. Evtikhiev", "E. Bogomolov", "Y. Sokolov", "T. Bryksin"],
       "year": 2023,
       "relevance": "Critical assessment of code generation evaluation metrics, directly relevant to methodology quality in AI code generation research."
     },
     {
       "title": "OPT: Open pre-trained transformer language models",
       "authors": ["S. Zhang", "S. Roller", "N. Goyal"],
       "year": 2022,
       "relevance": "Open-source LLM family used in this study, relevant to evaluation of open-source code generation models."
     },
     {
       "title": "LLaMA: Open and efficient foundation language models",
       "authors": ["H. Touvron", "T. Lavril", "G. Izacard"],
       "year": 2023,
       "relevance": "Foundation model family (Llama) used in this study, central to open-source LLM capability evaluation."
     }
   ]
 }