ai-research-survey

scan.json (29322B)

---

 {
   "paper": {
     "title": "GeoCode-GPT: A Large Language Model for Geospatial Code Generation Tasks",
     "authors": [
       "Shuyang Hou",
       "Zhangxiao Shen",
       "Anqi Zhao",
       "Jianyuan Liang",
       "Zhipeng Gui",
       "Xuefeng Guan",
       "Rui Li",
       "Huayi Wu"
     ],
     "year": 2024,
     "venue": "International Journal of Applied Earth Observation and Geoinformation",
     "arxiv_id": "2410.17031",
     "doi": "10.1016/j.jag.2025.104456"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GeoCode-GPT-7B, fine-tuned from Code Llama-7B on domain-specific geospatial corpora using QLoRA and LoRA, outperforms the base model by 27.6% average on multiple-choice questions, 21.4% on code summarization, and 25.1% on code generation across the GeoCode-Eval benchmark. The model approaches GPT-4 performance on most metrics despite having far fewer parameters, though it lags behind GPT-4 on entity recognition, platform knowledge, and code readability. The open-sourced GeoCode-PT (275K code snippets), GeoCode-SFT (502K instructions), and GeoCode-Eval (4,000 items) represent the first dedicated geospatial code generation corpus and benchmark.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL is provided: https://github.com/whuhsy/GeoCode-GPT. The paper states the corpora and model are open-sourced at this link."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states GeoCode-PT, GeoCode-SFT, and GeoCode-Eval are open-sourced at the same GitHub repository. The contributions section explicitly states 'We developed and open-sourced the GeoCode-PT and GeoCode-SFT corpora, as well as the GeoCode-Eval evaluation set.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two NVIDIA A100 40GB GPUs' and specific hyperparameters, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions. No software dependency versions are listed."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The training process is described at a high level (Sections 4.2-4.3) with hyperparameters, but there are no specific commands or scripts to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 5, 6, and 7 report only point estimates (e.g., 0.848 average accuracy). No confidence intervals, error bars, or uncertainty measures are reported for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GeoCode-GPT 'outperforms' other models across all metrics based solely on comparing raw numbers. No statistical significance tests (t-tests, bootstrap, etc.) are used for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context throughout — e.g., 'outperforms other models in multiple-choice accuracy by 9.1% to 32.1%' and tables show both absolute scores and differences from GeoCode-GPT for each model, allowing readers to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation set sizes (3,000 MC questions, 500 code generation, 500 summarization) are stated but no justification is given for why these sizes are sufficient. No power analysis or sample size rationale is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or any spread measure is reported. All results appear to be from single runs with no indication of stability across runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive baselines are included (Section 5): GPT-4, GPT-3.5, ERNIE 4.0, LLaMA 2-7B, LLaMA 3-8B, CodeGemma-7B, StarCoder 2-7B, CodeGeeX 2-6B, Code Llama-13B, and the base model Code Llama-7B."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4, LLaMA 3-8B, CodeGemma-7B, and StarCoder 2-7B, which are contemporary models for 2024. The selection spans commercial and open-source models of various sizes."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (QLoRA pretraining on GeoCode-PT, LoRA fine-tuning on GeoCode-SFT, different corpus components) but no ablation study isolates the contribution of each component. There is no experiment showing pretraining-only vs fine-tuning-only, or different corpus subsets."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: 6 accuracy dimensions for MC questions, 3 metrics for code summarization (Completeness, Accuracy, Readability), and 3 metrics for code generation (Accuracy, Readability, Executability)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is included: experts run generated code on different platforms to assess executability, and experts perform blind ranking of code readability across models (Section 5.2.2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "GeoCode-Eval is a separate evaluation set constructed to prevent knowledge leakage: 'all multiple-choice questions in GeoCode-Eval were sourced from platforms such as Baidu Wenku, Niuke, and Daoke Wenku' and code tasks used 'different valid code snippets' from GeoCode-SFT."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-category breakdowns are provided: Table 5 shows 6 MC dimensions individually, Table 6 breaks down 3 summarization metrics, and Table 7 breaks down 3 code generation metrics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes where GeoCode-GPT underperforms GPT-4 (e.g., entity recognition, platform knowledge) but does not show specific failure examples, error analysis, or qualitative examination of where the model produces incorrect code."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. The paper does not describe any approaches tried and abandoned, configurations that failed, or design decisions that hurt performance. All reported experiments show improvement over the baseline."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '9.1% to 32.1%' MC improvement, '1.7% to 25.4%' summarization improvement, and '1.2% to 25.1%' code generation improvement are supported by Tables 5, 6, and 7 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the QLoRA+LoRA training strategy 'achieves significant improvements' and attributes gains to their corpus and fine-tuning approach. While the before/after comparison with Code Llama-7B supports that some change occurred, the lack of ablation means the causal contribution of each component (QLoRA pretraining, LoRA fine-tuning, specific corpus elements) is not isolated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are consistently bounded to 'geospatial code generation tasks' throughout the paper. The title, abstract, and conclusion frame results specifically within the geospatial domain, and they don't claim general code generation superiority."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider confounds such as evaluation data overlap with training sources, GPT-4 judge bias, or whether memorization of platform-specific syntax (rather than genuine understanding) drives improvements."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures MC accuracy, GPT-4-scored summarization, and expert-evaluated code generation, framing these as demonstrating capability in 'geospatial code generation.' However, it does not discuss whether these proxies (especially MC questions and GPT-4 scores) actually capture real-world geospatial coding ability or productivity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referenced by general names: 'GPT-4', 'GPT-3.5', 'Code Llama-7B', 'LLaMA 2-7B', etc. No specific version identifiers, snapshot dates, or API versions are provided for any model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompt templates for GPT-4 evaluation scoring are provided in Fig. 6 (summarization) and Fig. 7 (code generation). The GeoCode-SFT instruction format (Instruct-Input-Output) is described in Section 3.2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters are reported in Sections 4.2 and 4.3: batch size 64, int4 quantization, learning rate 0.0002, cosine decay, weight decay 0.1, gradient accumulation 4, sequence length 4096, LoRA rank 64, NF4, scaling factor 128, dropout 0.05."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are prompted directly for evaluation tasks without any agent workflow, tool use, or iterative scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides detailed documentation of data construction: code collection from specific platforms (GitHub, Stack Overflow, Hugging Face), syntax screening, structured extraction algorithms (Fig. 3), Self-Instruct generation process (Fig. 4), and quality checking. Tables 1-4 enumerate all data components."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 'Limitations' is a dedicated subsection discussing the gap with GPT-4 and room for improvement in executability."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.1 discusses only future improvements ('expanding the scale of instruction data', 'leveraging more powerful hardware') rather than specific threats to the validity of current results. No discussion of evaluation bias, benchmark specificity, or GPT-4-as-judge reliability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of which geospatial platforms are not covered, what types of geospatial tasks are excluded, or limitations of evaluating only on GeoCode-Eval."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GeoCode-PT, GeoCode-SFT, and GeoCode-Eval datasets are stated to be open-sourced at the GitHub repository, making the underlying data available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides detailed descriptions of data collection: code from GitHub/Stack Overflow/Hugging Face, operator knowledge from official documentation, dataset knowledge from platform pages, encyclopedic documents from Wikipedia. Table 1 gives complete inventory with quantities and sources."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Expert evaluators are used for executability testing and readability ranking but no details are provided about who these experts are, how many were involved, how they were selected, or their qualifications."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from raw data collection through structured extraction (Fig. 3), Self-Instruct generation (Fig. 4), to final corpus assembly is documented with quantities at each stage (Tables 1-4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: State Key Laboratory of Information Engineering in Surveying, Mapping and Remote Sensing, Wuhan University; and School of Remote Sensing and Information Engineering, Wuhan University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence of the funder cannot be assessed. The work appears to be academic but funding is not mentioned."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any model evaluated. Code Llama's training cutoff is not mentioned, nor are the cutoffs for GPT-4, GPT-3.5, or other baselines."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions sourcing MC questions from external platforms 'to ensure fairness in evaluation and prevent knowledge leakage,' but does not analyze whether the baseline models (GPT-4, GPT-3.5, etc.) may have seen GeoCode-Eval source data during their training. The overlap prevention is only for GeoCode-GPT's own training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GeoCode-Eval sources questions from public platforms (Baidu Wenku, Niuke, Daoke Wenku) that could be in commercial models' training data. The paper does not discuss this contamination risk for baseline models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper has no human subjects study. Expert evaluators serve as judges, not experimental participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study — expert judges are not experimental participants requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Expert judges are used but this is not a human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper does mention 'blind selection process' for readability ranking, but this is a judge evaluation method, not an experimental design element for human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported for any model. The token costs of using GPT-4 as evaluator are mentioned in passing ('Given the token cost of LLMs') but not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('two NVIDIA A100 40GB GPUs') and training hyperparameters are detailed, but total training time, GPU hours, or total compute budget is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented as point estimates with no indication of how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are reported in detail but no search budget, search method, or number of configurations tried is described. The choices appear to be presented without justification of the search process."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The base model selection (Code Llama-7B) is justified as balancing 'performance, flexibility, and training costs' (Section 4.1), but the specific hyperparameter configuration is presented without justification for why these values were selected over alternatives."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across 11 models and multiple dimensions without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own model on their own benchmark (GeoCode-Eval) constructed by the same team. This self-evaluation bias is not acknowledged or discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares a 7B model against GPT-4 and notes the parameter gap ('despite having significantly fewer parameters than commercial models') but does not report performance as a function of compute budget or compare at matched compute levels."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "GeoCode-Eval is framed using Bloom's taxonomy (Fig. 5) as a conceptual basis, but there is no empirical validation that the benchmark actually measures real-world geospatial code generation ability. No comparison with alternative benchmarks or external validity analysis."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the model comparisons. Models are prompted directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. GeoCode-Eval MC questions from public platforms (Baidu Wenku, Niuke) could exist in commercial models' training data from before their training cutoffs, but this is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The MC question format could give models hints through option structure, and the code tasks include metadata that could leak information, but this is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper states GeoCode-Eval uses 'different valid code snippets' from GeoCode-SFT, but does not formally verify independence or measure similarity between training and evaluation data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. The paper mentions sourcing evaluation questions from external platforms to 'prevent knowledge leakage' but does not use canary strings, membership inference, n-gram overlap analysis, or other detection techniques."
       }
     }
   },
   "claims": [
     {
       "claim": "GeoCode-GPT-7B outperforms Code Llama-7B by an average of 27.6% on multiple-choice accuracy across six geospatial knowledge dimensions.",
       "evidence": "Table 5 shows GeoCode-GPT-7B achieving 0.848 average accuracy vs Code Llama-7B at 0.572, with improvements ranging from 0.08 (PLR) to 0.508 (DK). Section 5.1.",
       "supported": "moderate"
     },
     {
       "claim": "GeoCode-GPT-7B improves code summarization by 21.4% overall compared to Code Llama-7B, approaching GPT-4 performance.",
       "evidence": "Table 6 shows GeoCode-GPT-7B at 0.914 overall vs Code Llama-7B at 0.700. GPT-4 scores 0.897, which GeoCode-GPT surpasses slightly. Based on GPT-4-as-judge scoring. Section 5.2.1.",
       "supported": "moderate"
     },
     {
       "claim": "GeoCode-GPT-7B improves code generation by 25.1% overall compared to Code Llama-7B.",
       "evidence": "Table 7 shows GeoCode-GPT-7B at 0.636 overall vs Code Llama-7B at 0.385. Includes expert-assessed executability (0.504 vs 0.302). Section 5.2.2.",
       "supported": "moderate"
     },
     {
       "claim": "GeoCode-GPT-7B is the first LLM specifically designed for geospatial code generation tasks.",
       "evidence": "Stated in the abstract and Section 1 contributions. The related work section (Section 2) confirms no prior domain-specific LLM for geospatial code exists.",
       "supported": "moderate"
     },
     {
       "claim": "The QLoRA+LoRA training strategy effectively balances computational resources and training efficiency for domain adaptation.",
       "evidence": "The paper describes the strategy (Sections 4.2-4.3) and shows improved results over the base model, but provides no ablation isolating QLoRA pretraining vs LoRA fine-tuning contributions, and no comparison with alternative training strategies.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Self-evaluation on own benchmark",
       "detail": "GeoCode-GPT is trained on GeoCode-PT/SFT and evaluated on GeoCode-Eval, all constructed by the same team using similar sources and methods. The proximity between training and evaluation data raises contamination risk. The paper does not acknowledge this self-evaluation bias."
     },
     {
       "flag": "No statistical tests or variance reporting",
       "detail": "All claims of outperformance are based on comparing raw point estimates without significance tests, confidence intervals, or variance across runs. Results appear to be single-run. Differences as small as 0.008 (GPT-3.5 code generation accuracy) are presented as improvements."
     },
     {
       "flag": "GPT-4 as judge while GPT-4 is a baseline",
       "detail": "GPT-4 is used as the automatic evaluator for code summarization scoring while simultaneously being one of the baseline models. This introduces potential bias — GPT-4 may systematically favor certain output styles, including its own."
     },
     {
       "flag": "No ablation study for multi-component system",
       "detail": "The system involves QLoRA pretraining, LoRA fine-tuning, and multiple corpus components (code docs, operator knowledge, dataset knowledge, encyclopedic docs, Alpaca data) but no ablation isolates individual contributions."
     },
     {
       "flag": "Evaluation data sourced from public platforms",
       "detail": "MC questions are sourced from Baidu Wenku, Niuke, and Daoke Wenku — public educational platforms whose content may be in commercial models' training data. This could systematically disadvantage or advantage certain models."
     }
   ],
   "cited_papers": [
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Foundation model for code generation; base model for GeoCode-GPT fine-tuning."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming--The Rise of Code Intelligence",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Open-source code generation LLM used as baseline context for code generation capabilities."
     },
     {
       "title": "Wizardcoder: Empowering code large language models with evol-instruct",
       "authors": ["Z. Luo", "C. Xu", "P. Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Code generation LLM using instruction evolution for training, relevant to code LLM fine-tuning approaches."
     },
     {
       "title": "Lora: Low-rank adaptation of large language models",
       "authors": ["E.J. Hu", "Y. Shen", "P. Wallis"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Foundational PEFT method widely used for LLM fine-tuning."
     },
     {
       "title": "Self-instruct: Aligning language models with self-generated instructions",
       "authors": ["Y. Wang", "Y. Kordi", "S. Mishra"],
       "year": 2022,
       "arxiv_id": "2212.10560",
       "relevance": "Framework for synthetic instruction data generation used in GeoCode-SFT construction."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and pass@k metric for code generation evaluation."
     },
     {
       "title": "ICE-Score: Instructing Large Language Models to Evaluate Code",
       "authors": ["T.Y. Zhuo"],
       "year": 2023,
       "arxiv_id": "2304.14317",
       "relevance": "LLM-as-judge approach for code evaluation; relevant to automated code quality assessment."
     },
     {
       "title": "A survey on evaluating large language models in code generation tasks",
       "authors": ["L. Chen", "Q. Guo", "H. Jia"],
       "year": 2024,
       "arxiv_id": "2408.16498",
       "relevance": "Survey of evaluation methods for LLM code generation, directly relevant to code evaluation methodology."
     },
     {
       "title": "LIMA: Less Is More for Alignment",
       "authors": ["C. Zhou", "P. Liu", "P. Xu"],
       "year": 2023,
       "arxiv_id": "2305.11206",
       "relevance": "Demonstrates effectiveness of small high-quality instruction datasets for LLM alignment."
     },
     {
       "title": "Phi-3 technical report: A highly capable language model locally on your phone",
       "authors": ["M. Abdin", "S.A. Jacobs", "A.A. Awan"],
       "year": 2024,
       "arxiv_id": "2404.14219",
       "relevance": "Demonstrates effectiveness of synthetic training data for small LLMs."
     },
     {
       "title": "Systematic Evaluation of LLM-as-a-Judge in LLM Alignment Tasks: Explainable Metrics and Diverse Prompt Templates",
       "authors": ["H. Wei", "S. He", "T. Xia"],
       "year": 2024,
       "arxiv_id": "2408.13006",
       "relevance": "Evaluates LLM-as-a-Judge methodology, relevant to understanding reliability of GPT-4-based code evaluation."
     },
     {
       "title": "A Survey on Large Language Models for Code Generation",
       "authors": ["J. Jiang", "F. Wang", "J. Shen"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Comprehensive survey of LLMs for code generation."
     }
   ]
 }