ai-research-survey

scan.json (31117B)

---

 {
   "paper": {
     "title": "An efficient strategy for fine-tuning large language models",
     "authors": [
       "Benjamin Marsh",
       "Adam Michaleas",
       "Darrell O. Ricke",
       "Shaun Monera",
       "Shriya Zembruski"
     ],
     "year": 2026,
     "venue": "Frontiers in Artificial Intelligence",
     "doi": "10.3389/frai.2026.1665992"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DSS combined with full-precision fine-tuning yields the strongest performance for NL-to-Query DSL translation across FLAN-T5 model sizes, but at higher GPU memory cost for smaller models. Under resource constraints, LoRA and QLoRA with a 4:1 alpha-to-rank ratio provide effective performance-efficiency tradeoffs. Counter-intuitively, LoRA and QLoRA used more GPU memory than full-precision for FLAN-T5 Large due to adapter overhead and implementation differences. DSS rationales consistently improved training over label-only supervision across all 8 tested configurations in ablation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A public GitHub repository is provided: https://github.com/brmarsh23/An-Efficient-Strategy-for-Fine-Tuning-Large-Language-Models. The paper states 'The code and instructions are available at the following Git Repository link' (Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The Data Availability Statement explicitly states: 'The datasets presented in this article are not readily available because dataset utilized in the submission is Controlled Unclassified Information (CUI) from US Department of Defense computer information systems.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.5 describes hardware (Intel Xeon Platinum 8480+, NVIDIA H100 GPUs, 2 TB RAM) and names libraries (PyTorch, PEFT, bitsandbytes, Ray Train) but provides no library versions, requirements.txt, or Dockerfile in the paper itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper itself contains no step-by-step reproduction instructions. A GitHub URL is provided with claimed 'instructions,' but the dataset is CUI-restricted, making full reproduction impossible regardless of code availability."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 3 and 4 report point estimates of evaluation loss with no confidence intervals, error bars, or uncertainty measures. The ablation (Table 4) is averaged over 2 seeds but reports no CI or spread."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims full-precision 'yields the strongest overall performance' and DSS 'consistently improves model fine-tuning' but provides no statistical significance tests (no p-values, t-tests, or bootstrap tests) to support any comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 shows raw loss differences between DSS and label-only training (e.g., +1.4e-3) but no standardized effect sizes (Cohen's d, percentage improvement, or relative differences). Main results in Table 3 are raw loss values with no effect size framing."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset consists of 1,000 questions with no justification for this sample size, no power analysis, and no discussion of whether 1,000 examples is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Table 3) are single-run best evaluation losses. The ablation (Table 4) is 'averaged over two random seeds' but reports no standard deviation, IQR, or spread measure. The reader cannot assess result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three fine-tuning methods are systematically compared: full-precision, LoRA, and QLoRA. The ablation additionally compares DSS (α=0.5) against label-only (α=1.0) training."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LoRA (Hu et al., 2021) and QLoRA (Dettmers et al., 2023) are the current standard parameter-efficient fine-tuning methods. DSS (Hsieh et al., 2023) is recent. All are appropriate contemporary methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.7 describes a dedicated ablation study comparing DSS rationale-augmented training (α=0.5) versus label-only training (α=1.0) 'across model sizes and fine-tuning methods.' Results in Table 4."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports evaluation loss for task performance, plus GPU memory usage, training samples per second, and total training time as efficiency metrics (Figure 6, Table 3). Multiple dimensions of comparison are provided."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is performed. The paper fine-tunes models for NL-to-Query DSL translation but never has humans assess whether the generated DSL queries are correct, usable, or semantically appropriate."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The 80/20 split produces an 'evaluation dataset' that is used both for learning rate reduction decisions ('evaluation loss was monitored in order to conduct learning rate reduction after 10 epochs of no improvement,' Section 3.5) and for final performance reporting. This makes it a validation set, not a held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model architecture (Small/Base/Large/XL), fine-tuning method (full-precision/LoRA/QLoRA), and hyperparameter settings (Rank/Alpha combinations). Figure 6 provides a quad chart; Table 3 lists top 8 models; Figure 7 breaks down by Rank/Alpha."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative error analysis of model outputs is provided. The paper does not show examples where the fine-tuned model produced incorrect Query DSL or discuss specific failure modes of the generated outputs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports the counter-intuitive finding that 'LoRA and QLoRA methods required significantly more memory to train the highest-performing model type, the FLAN-T5 Large, than the full-precision method' (Section 4), contradicting theoretical expectations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: DSS + full-precision strongest (Table 3, loss 0.06384), LoRA provides performance-efficiency tradeoff (Table 3, Section 5.1), QLoRA enables larger models under memory constraints (only method to run FLAN-T5 XL), 4:1 alpha-to-rank ratio (Figure 7, Section 5.2)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary causal claim — that DSS rationales improve training — is supported by a controlled ablation (Section 3.7, Table 4) that manipulates only α (0.5 vs 1.0) while holding all other hyperparameters constant. This single-variable manipulation is adequate for causal inference within the tested setting."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'fine-tuning large language models' broadly, and the abstract proposes 'a general guide for efficiently fine-tuning LLMs for domain-specific tasks.' However, the study tests only one task (NL to Query DSL) with one model family (FLAN-T5 encoder-decoder). The Limitations section acknowledges this but the title and abstract still overclaim."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses alternative explanations for the counter-intuitive memory findings: additional adapter overhead, dequantization costs during forward/backward passes, and 'implementation differences between the full-precision and LoRA/QLoRA methods' (using standard PyTorch vs PEFT/bitsandbytes libraries)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 explicitly acknowledges: 'the metrics do not directly capture task-level correctness, such as exact match rates on the DSL JSON' and recommends 'future work should incorporate additional metrics, such as BLEU, METEOR, and TER scores.' This directly addresses the gap between measured proxy (token-level loss) and claimed outcome (task effectiveness)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model architectures are identified: FLAN-T5 Small (76.9M), Base (247.6M), Large (770.6M), and XL (2,884.5M) with detailed architecture parameters in Table 1. Teacher model specified as Mixtral 8x22B. These are well-defined open-source models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows an example input prompt with its three-part structure: DSL interface instructions, dataset description, and Chain-of-Thought prompting. Figure 3 shows the dataset creation process. Figure 4 shows the training step format with task prefixes."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports hyperparameters: learning rate (5e-5), patience (10 epochs), LR factor (1e-1), epochs (100), batch size (8), alpha (0.5). Section 3.6 details LoRA/QLoRA-specific parameters: Rank values (32, 64, 128), Alpha values, target modules, and dropout rate."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The work involves standard model fine-tuning with DSS, not agentic workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.2 describe the data pipeline: 1,000 NL questions processed through Mixtral 8x22B with chain-of-thought prompting to generate labels and rationales, then formatted as multi-task training data with task prefixes, and split 80/20 for training and evaluation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 is titled 'Limitations' and contains two substantial paragraphs discussing multiple specific limitations of the study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 lists specific threats: single downstream task (NL to Query DSL), token-level loss without task-level metrics, FLAN-T5 only without decoder-only architectures, limited random seeds, and incomplete hyperparameter exploration due to compute constraints."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 explicitly states: 'conclusions may not directly transfer to other domains, such as open-ended text generation, conversational dialogue, or classification tasks,' and 'the methodology focuses on the FLAN-T5 encoder-decoder family and does not include decoder-only architectures.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The dataset is classified as Controlled Unclassified Information (CUI) and 'not readily available.' Requests must be directed to benjamin.marsh@usmc.mil. Independent verification of the underlying data is not possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes: 1,000 NL questions designed for querying organizational data, translated to Query DSL using Mixtral 8x22B as teacher model via chain-of-thought prompting. The prompt structure is shown in Figure 2."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The origin of the 1,000 input natural language questions is never described. Were they hand-crafted, sampled from query logs, or synthetically generated? The paper jumps from describing the task to describing the teacher model output without explaining where the input questions came from."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from input questions through teacher-model generation to fine-tuning format is documented across Sections 3.1-3.2 and Figures 3-4. The paper mentions 'all training examples were screened for correctness and validity' (Section 5.5), though screening criteria are not detailed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding section states: 'This material is based upon work supported by the Department of the Air Force under Air Force Contract No. FA8702-15-D-0001.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Marine Corps Tactical Systems Support Activity (USMC) and MIT Lincoln Laboratory, Artificial Intelligence Technology. No conflict with evaluated products exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (US Department of the Air Force) has no commercial interest in any of the evaluated methods or models (FLAN-T5, LoRA, QLoRA are open-source methods from academic/industry research). The funder benefits from knowing which method works, not from a specific outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The Conflict of Interest section states: 'The author(s) declared that this work was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. Models are fine-tuned on a custom dataset (NL to Query DSL) created by the authors; contamination from pre-training data is not a relevant concern for this evaluation setup."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the evaluation dataset was generated by the authors specifically for this study, not drawn from a public benchmark. There is no pre-trained model benchmark evaluation scenario."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No public benchmark is used. The evaluation is on a custom CUI dataset, so standard benchmark contamination concerns do not apply."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a model fine-tuning study with automated evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported. The paper focuses entirely on training costs. How much it costs to run the fine-tuned models at inference time is not discussed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total compute time is stated: 'The total compute time for the hyperparameter search was 499.6 hours' (Section 3.6). Hardware is detailed: two-node cluster with four NVIDIA H100 80GB GPUs per node. Per-run training times are in Table 3."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main hyperparameter search results (Table 3) are single-run best evaluation losses with no seed variation. The ablation (Table 4) averages over only 2 random seeds with no spread reported. The paper acknowledges 'ablation results are averaged over a limited number of random seeds' (Section 5.5)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 explicitly states: '3 runs were performed using full-precision fine-tuning, 39 runs were performed with the LoRA fine-tuning method, and 44 runs with the QLoRA fine-tuning method.' Total of 86 hyperparameter sweeps. Ablation states 'averaged over two random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive search budget reported: 86 total sweeps, 499.6 compute hours, with explicit enumeration of Rank values (32, 64, 128), Alpha values per Rank, and per-architecture/per-method run counts (Section 3.6)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.8 states: 'the best evaluation loss achieved for each hyperparameter sweep was used to perform the final comparison.' Selection criterion is clear and all top 8 configurations are reported in Table 3 rather than just the single best."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "86 hyperparameter configurations are compared with no correction for multiple comparisons. Claims of 'best' and 'consistent' patterns are made without adjusting for the number of comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss author-evaluation bias. While they use standard library implementations rather than custom re-implementations, they do not acknowledge that implementation choices (e.g., which PyTorch vs PEFT optimizations are used) could systematically advantage one method."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 explicitly plots performance (evaluation loss) alongside GPU memory usage, training samples per second, and total training time for each method and model size. Table 3 also pairs loss with GPU usage and training time. This is a central contribution of the paper."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 explicitly discusses construct validity: 'the metrics do not directly capture task-level correctness, such as exact match rates on the DSL JSON' and recommends additional metrics (BLEU, METEOR, TER). This directly questions whether the measured metric (token loss) captures what matters (task correctness)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The study compares fine-tuning methods directly, not scaffold-dependent systems."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the FLAN-T5 pre-training data or Mixtral 8x22B training data could contain Query DSL patterns or similar structured generation examples that would advantage the models on this task."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the DSS multi-task training setup (where models learn to generate both rationales and labels) could leak information between tasks at evaluation time, or whether the teacher-generated labels could embed patterns from the teacher's training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 80/20 train-eval split is applied to 1,000 questions without discussion of whether train and eval examples share structural patterns (e.g., similar question templates, overlapping Query DSL structures) that would inflate performance estimates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No overlap analysis, deduplication, or independence verification between train and evaluation splits."
       }
     }
   },
   "claims": [
     {
       "claim": "DSS combined with full-precision fine-tuning yields the strongest overall performance across model architectures.",
       "evidence": "Table 3 shows full-precision FLAN-T5 Large achieves lowest evaluation loss (0.06384), outperforming all LoRA and QLoRA configurations. Figure 6 top-left shows full-precision has lower mean evaluation loss across all architectures.",
       "supported": "moderate"
     },
     {
       "claim": "A LoRA Alpha-to-Rank ratio of 4:1 provides optimal performance for parameter-efficient fine-tuning.",
       "evidence": "Figure 7 shows average evaluation loss decreasing with increasing Alpha relative to Rank, with peak performance at 4:1 ratio. Table 3 shows top LoRA model uses Rank 128, Alpha 512 (4:1 ratio).",
       "supported": "moderate"
     },
     {
       "claim": "DSS rationales consistently improve model training over label-only supervision across all model sizes and fine-tuning methods.",
       "evidence": "Table 4 shows all 8 configurations (3 model sizes × 3 methods, minus FLAN-T5 Large full-precision) achieve lower evaluation loss with DSS (α=0.5) than label-only (α=1.0). Differences range from +2.5e-4 to +1.6e-2.",
       "supported": "moderate"
     },
     {
       "claim": "QLoRA enables fine-tuning of larger models (FLAN-T5 XL) under GPU memory constraints that prevent full-precision and LoRA training.",
       "evidence": "Section 3.6 states only QLoRA could run FLAN-T5 XL within available GPU memory. Table 3 shows FLAN-T5 XL QLoRA achieving competitive loss (0.06874, rank 3 overall).",
       "supported": "strong"
     },
     {
       "claim": "LoRA and QLoRA can require more GPU memory than full-precision fine-tuning at larger model sizes.",
       "evidence": "Figure 6 bottom-left shows for FLAN-T5 Large, full-precision uses the least average GPU memory, while LoRA and QLoRA use more. Section 4 explains this via adapter overhead, dequantization costs, and implementation differences.",
       "supported": "strong"
     },
     {
       "claim": "Rationale supervision has the largest impact when model capacity and update precision are most constrained.",
       "evidence": "Table 4 shows the largest loss improvement from DSS occurs with FLAN-T5 Small QLoRA (+1.6e-2), while the smallest is FLAN-T5 Base LoRA (+2.5e-4). Section 5.3 interprets this as rationales compensating for limited model capacity.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-task evaluation with broad claims",
       "detail": "All conclusions are drawn from one translation task (NL to Query DSL) with one model family (FLAN-T5), but the paper's title claims 'fine-tuning large language models' broadly and the abstract proposes 'a general guide.' The generalization gap between what was tested and what is claimed is substantial."
     },
     {
       "flag": "No task-level correctness evaluation",
       "detail": "Only token-level cross-entropy loss is reported. No exact match, BLEU, METEOR, or semantic correctness metrics are used. The paper acknowledges this in limitations but draws conclusions about method effectiveness without knowing whether generated Query DSL outputs are actually correct or usable."
     },
     {
       "flag": "Validation set used as test set",
       "detail": "The 80/20 evaluation split was used for learning rate scheduling decisions (LR reduction after 10 epochs of no improvement) and simultaneously as the final evaluation metric. This conflates validation and testing, potentially inflating reported performance."
     },
     {
       "flag": "No error bars or significance tests",
       "detail": "Main results are single-run best losses. Even the ablation uses only 2 seeds with no spread reported. Comparative claims ('yields the strongest performance,' 'consistently improves') are made without any statistical testing."
     },
     {
       "flag": "Restricted dataset prevents verification",
       "detail": "The dataset is classified as CUI and unavailable to the public. Independent researchers cannot verify the data quality, check for issues in teacher-generated labels/rationales, or replicate the experiments."
     }
   ],
   "cited_papers": [
     {
       "title": "Distilling step-by-step! Outperforming larger language models with less training data and smaller model sizes",
       "authors": ["C.-Y. Hsieh", "C.-L. Li", "C.-K. Yeh", "H. Nakhost", "Y. Fujii", "A. Ratner"],
       "year": 2023,
       "arxiv_id": "2305.02301",
       "doi": "10.18653/v1/2023.findings-acl.507",
       "relevance": "Core method used in this paper — knowledge distillation with chain-of-thought rationales to train smaller models efficiently."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis", "Z. Allen-Zhu", "Y. Li", "S. Wang"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning method benchmarked in this paper; foundational PEFT technique for LLMs."
     },
     {
       "title": "QLoRA: Efficient Finetuning of Quantized LLMs",
       "authors": ["T. Dettmers", "A. Pagnoni", "A. Holtzman", "L. Zettlemoyer"],
       "year": 2023,
       "arxiv_id": "2305.14314",
       "relevance": "Quantized parameter-efficient fine-tuning method benchmarked in this paper; enables training larger models under memory constraints."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans", "M. Bosma", "B. Ichter", "F. Xia"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "relevance": "Prompting methodology used for eliciting rationales from the teacher model in the DSS pipeline."
     },
     {
       "title": "Parameter-efficient fine-tuning for large models: a comprehensive survey",
       "authors": ["Z. Han", "C. Gao", "J. Liu", "J. Zhang", "S. Q. Zhang"],
       "year": 2024,
       "arxiv_id": "2403.14608",
       "relevance": "Survey of PEFT methods relevant to understanding the landscape of efficient fine-tuning approaches."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Addresses cost-efficiency of LLM usage, directly relevant to the cost-constrained fine-tuning focus of this paper."
     },
     {
       "title": "Exploring the limits of transfer learning with a unified text-to-text transformer",
       "authors": ["C. Raffel", "N. Shazeer", "A. Roberts", "K. Lee", "S. Narang", "M. Matena"],
       "year": 2023,
       "arxiv_id": "1910.10683",
       "relevance": "T5 architecture foundation paper — the model family (FLAN-T5) evaluated in this work."
     },
     {
       "title": "Finetuned language models are zero-shot learners",
       "authors": ["J. Wei", "M. Bosma", "V. Zhao", "K. Guu", "A. W. Yu", "B. Lester"],
       "year": 2022,
       "relevance": "Instruction tuning methodology that produced the FLAN-T5 models used in this study."
     },
     {
       "title": "Capabilities of GPT-4 on medical challenge problems",
       "authors": ["H. Nori", "N. King", "S. M. McKinney", "D. Carignan", "E. Horvitz"],
       "year": 2023,
       "arxiv_id": "2303.13375",
       "relevance": "Demonstrates domain-specific LLM evaluation challenges, motivating the need for fine-tuning approaches like those studied here."
     },
     {
       "title": "Optimizing large language models with an enhanced LoRA fine-tuning algorithm for efficiency and robustness in NLP tasks",
       "authors": ["J. Hu", "X. Liao", "J. Gao", "Z. Qi", "H. Zheng", "C. Wang"],
       "year": 2024,
       "arxiv_id": "2412.18729",
       "relevance": "Recent work on enhanced LoRA fine-tuning, directly related to the parameter-efficient methods evaluated here."
     }
   ]
 }