ai-research-survey

scan.json (27236B)

---

 {
   "paper": {
     "title": "Evaluation of Code LLMs on Geospatial Code Generation",
     "authors": [
       "Piotr Gramacki",
       "Bruno Martins",
       "Piotr Szymański"
     ],
     "year": 2024,
     "venue": "GeoAI@SIGSPATIAL",
     "arxiv_id": "2410.04617",
     "doi": "10.1145/3687123.3698286"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "StarCoder2-7b achieves the best geospatial code generation performance (32.47% pass@1) among seven 7B/8B-parameter LLMs, despite ranking 4th on HumanEval, showing that generic code benchmarks do not predict domain-specific performance. Multi-step geospatial tasks are substantially harder than single-step tasks across all models. Models fail almost completely on OSMNX and MovingPandas libraries while performing reasonably on Shapely and H3, revealing significant gaps in domain-specific tool knowledge. Task framing (operation vs. semantic) has a small mixed effect, suggesting weak geospatial reasoning in current models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/kraina-ai/geospatial-code-llms-dataset (footnote 1). The abstract states 'We share our dataset and reproducible evaluation code on a public GitHub repository.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset of 77 samples is shared on the same GitHub repository. The abstract confirms the dataset is publicly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using the transformers library, bitsandbytes for 4-bit quantization, and specific GPU hardware, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the evaluation pipeline methodology (Section 4.1) including code trimming, import handling, and testing procedure, but does not provide step-by-step commands or a reproduction guide. The GitHub repo is referenced but the paper itself lacks concrete instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-7 are reported as single point estimates (e.g., '32.47%') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'best results have been achieved by starcoder2-7b', 'models work better with geodataframes') based solely on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw percentages in tables. While HumanEval scores are shown alongside benchmark scores for context, no formal effect sizes (Cohen's d, relative improvement, etc.) are computed or discussed."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 77 samples from 20 unique tasks. No justification is given for why this sample size is sufficient, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments use greedy decoding producing a single deterministic output per sample. No variance, standard deviation, or spread measures are reported across any experimental condition."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven models are compared against each other, and HumanEval scores from external leaderboards are included as reference baselines (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models include StarCoder2, CodeLlama, Llama 3, Mistral 7B, Gemma, and CodeGemma — all released in 2023-2024 and representing contemporary open-source code LLMs at the 7B/8B scale."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The contribution is a benchmark dataset, not a system with components to ablate. The multi-dimensional analysis (complexity, framing, input format, tools) serves a similar diagnostic purpose but is not an ablation study."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: Accuracy (percentage of passed test cases), Pass@1 (percentage of fully correct solutions), and Pass_any@1 (percentage of solutions passing at least one test case). Described in Section 4.1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via test case pass/fail. No human evaluation of code quality, readability, or correctness is performed. The paper explicitly chose automated testing over reference-solution comparison (Section 3.4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The entire benchmark was hand-crafted specifically for this evaluation: 'The prompts are human-written to ensure that they were not present in any training data for existing models' (Section 3.2). No model tuning was done on this data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided across five dimensions: task complexity (Table 3), task framing (Table 4), input format (Table 5), tool usage (Table 6), and point format (Table 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses gemma/codegemma models hallucinating and generating repetitive code blocks, models generating placeholder functions with NotImplementedError for unknown libraries (Listing 4), and near-complete failure on OSMNX and MovingPandas."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results reported: all models fail on OSMNX and MovingPandas (Table 6), gemma models perform very poorly overall, models struggle with GeoJSON and Shapefile formats compared to GeoDataFrames, and HumanEval performance does not predict geospatial performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are modest and supported: they propose a benchmark (described in Section 3), test existing models (Section 4), and share code/data (GitHub link). No overclaiming is evident."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims about why models fail (e.g., 'models that are worse at geospatial code generation find it especially difficult to solve multi-step tasks,' suggesting task complexity causes performance drops) without controlling for confounds like task-specific difficulty or sample size per category."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its scope: 'Due to computational constraints we limited ourselves to 7B/8B scale LLMs, which we also aim to extend in the future' (Section 5). The title and abstract reference 'a selection of existing code generation LLMs' rather than making broad claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed results. For example, the poor performance on OSMNX/MovingPandas could be due to library documentation quality, training data volume, or task difficulty rather than just model capability, but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures functional correctness via test cases and frames its claims in terms of code correctness and task-solving capability. The claims match the measurement granularity — they don't overclaim 'productivity' or 'developer effectiveness' from pass@1 scores."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact HuggingFace model identifiers are provided for all 7 models (e.g., 'bigcode/starcoder2-7b', 'meta-llama/CodeLlama-7b-hf', 'meta-llama/Meta-Llama-3-8B'). These are specific, versioned model checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The exact prompt format is provided in Figure 1 and Listings 1-4, including function signatures, type hints, and docstrings. The format 'remains unchanged between tested models' (Section 3.2). All 77 prompts follow this template and are available in the GitHub repository."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are stated: greedy decoding (temperature=0), max_length=200, 4-bit quantization via bitsandbytes. Reported in Section 4.1 under 'Hyperparameters.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models generate code completions directly from function signatures via single-pass inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the evaluation pipeline: trimming responses to a single function by searching for the second 'def' string, searching generated code for library imports, installing them in a virtual environment, and importing before testing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 contains a dedicated 'Limitations' subsection discussing the benchmark's current scope and the computational constraints on model selection."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies specific limitations: 'the current version of the benchmark should be expanded to cover more typical tasks and tools,' and 'Due to computational constraints we limited ourselves to 7B/8B scale LLMs' (Section 5). These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what is not covered: no polygon inputs, no larger models, no fine-tuning experiments, no incorrect input handling, no code infilling tasks, limited to 4 tool libraries and 5 input formats. Stated in Sections 3 and 5."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The complete benchmark dataset (77 samples with prompts and test cases) is available on the public GitHub repository. Generated outputs could be verified against this data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the sample creation process: starting with task definition on the complexity dimension, then augmenting via input format and framing changes to produce variants. The categorization scheme is detailed in Section 3.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data is a manually created benchmark dataset, and the evaluated systems are publicly available LLMs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from task conception to final dataset is documented: define complexity dimension → select tools/input types → write task → augment across dimensions → create test cases (Section 3). The evaluation pipeline is documented in Section 4.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly stated: Wrocław University of Science and Technology (Gramacki, Szymański) and INESC-ID/Instituto Superior Técnico, University of Lisbon (Martins). Kraina.AI affiliation is also listed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. Without knowing the funding source, independence cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. The Kraina.AI affiliation is listed but no conflict-of-interest declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the 7 evaluated models. The paper relies on hand-crafted tasks as contamination mitigation but does not state when each model's training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'The prompts are human-written to ensure that they were not present in any training data for existing models' (Section 3.2). This directly addresses train/test overlap through benchmark design."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is addressed by design: the benchmark was manually created after the models' training periods, with human-written prompts to ensure novelty. This is stated in Section 3.2."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated benchmark testing of LLMs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on a code generation benchmark."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported. The paper mentions hardware used but not per-sample or total inference costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (GTX 1080 8GB, A100 80GB) but no total GPU hours, wall-clock time, or compute budget is quantified. Section 4.1 describes the machines but not how long experiments took."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy decoding produces deterministic outputs, so there is no seed sensitivity analysis. The paper does not explore whether results change with non-greedy sampling strategies."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states they use greedy decoding to produce 'a single output' per sample (Section 4.1 under 'Hyperparameters'), making the number of runs (1) explicit."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. The max_length=200 was 'verified to be enough' but no search budget or alternative configurations are discussed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "A uniform configuration is used across all models (greedy decoding, max_length=200, 4-bit quantization), eliminating configuration cherry-picking. The max_length was verified to be sufficient for all models and tasks."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluate models on it. No discussion of whether benchmark design choices might favor or disadvantage particular models, or acknowledgment of self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All models are 7B/8B parameters with the same 4-bit quantization, making compute differences negligible across the comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether 77 samples across 20 tasks adequately measure geospatial code generation capability, nor whether the four-dimensional categorization captures the full range of geospatial coding challenges."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models perform single-pass code completion from function signatures."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark tasks were manually created after the models' training periods. 'The prompts are human-written to ensure that they were not present in any training data for existing models' (Section 3.2). This is a temporal leakage prevention strategy."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the prompt format, function signatures, or docstring style could leak information about expected solutions. The evaluation pipeline also adds all needed imports (Section 4.1), which could make the task easier than real-world usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 77 samples are augmented from 20 unique tasks (Section 3.3), meaning many samples share the same underlying logic. This non-independence is not discussed as a potential source of inflated evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No formal leakage detection method (canary strings, membership inference, n-gram overlap analysis) is employed. The paper relies solely on manual creation as a prevention measure."
       }
     }
   },
   "claims": [
     {
       "claim": "Generic HumanEval performance does not predict geospatial code generation performance — StarCoder2-7b ranks 4th on HumanEval but 1st on the geospatial benchmark.",
       "evidence": "Table 2 shows StarCoder2-7b at 34.09% HumanEval but 32.47% pass@1 on geospatial tasks (best), while CodeLlama-Python has 40.48% HumanEval but only 24.68% geospatial pass@1.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-step geospatial tasks are significantly harder than single-step tasks for all code generation models.",
       "evidence": "Table 3 shows consistent drops across all models: StarCoder2 drops from 45.45% to 15.15%, CodeLlama from 29.55% to 12.12%, Gemma from 13.64% to 3.03%.",
       "supported": "moderate"
     },
     {
       "claim": "Current code LLMs fail almost completely on OSMNX and MovingPandas libraries.",
       "evidence": "Table 6: all models score 0% on MovingPandas, and 6 of 7 models score 0% on OSMNX (only StarCoder2 at 16.67%).",
       "supported": "strong"
     },
     {
       "claim": "Models perform better with GeoDataFrame inputs than Shapefile or GeoJSON formats.",
       "evidence": "Table 5 shows higher pass@1 for GeoDataFrame across most models (e.g., CodeLlama-hf: 27.27% gdf vs 18.75% shp vs 4.55% geojson).",
       "supported": "moderate"
     },
     {
       "claim": "Task framing (operation vs. semantic) has a small and inconsistent effect on model performance.",
       "evidence": "Table 4: most models perform slightly better with operation framing, but two models (Mistral, CodeLlama-hf) perform better with semantic framing. Differences are small (3-9 percentage points).",
       "supported": "moderate"
     },
     {
       "claim": "Models perform better with Shapely Point objects than raw latitude/longitude coordinates.",
       "evidence": "Table 7 shows consistent advantage for shapely format: StarCoder2 33.33% vs 66.67%, CodeLlama 4.76% vs 33.33%.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests or error bars",
       "detail": "All comparative claims (e.g., 'best model', 'models work better with') are based solely on comparing raw percentages without any statistical significance testing, confidence intervals, or error bars. With only 77 samples (many categories having 3-7 samples), observed differences could easily be due to chance."
     },
     {
       "flag": "Very small benchmark size",
       "detail": "The benchmark has only 77 samples from 20 unique tasks. Per-category breakdowns sometimes have as few as 3-4 samples (e.g., H3 with 3 tasks, OSMNX with 4 tasks), making percentage differences unreliable. The 77 samples are augmented from 20 unique tasks, so many share identical logic."
     },
     {
       "flag": "Single-run deterministic evaluation only",
       "detail": "Only greedy decoding (temperature=0) is used, producing a single output per sample. No exploration of sampling strategies (temperature, top-p) or pass@k with k>1, which are standard in code generation evaluation."
     },
     {
       "flag": "Non-independence of augmented samples",
       "detail": "The 77 samples are augmented from 20 unique tasks by varying input format and framing. Samples sharing the same underlying task are not independent, potentially inflating apparent benchmark coverage."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark for code generation evaluation, the primary reference baseline used in this paper."
     },
     {
       "title": "DS-1000: a natural and reliable benchmark for data science code generation",
       "authors": ["Yuhang Lai", "Chengxi Li"],
       "year": 2023,
       "relevance": "Domain-specific code generation benchmark for data science — directly comparable approach to the geospatial domain specialization in this paper."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "StarCoder2-7b was the best-performing model on the geospatial benchmark despite not being best on HumanEval."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "CodeLlama models evaluated in this study; foundational open-source code generation models."
     },
     {
       "title": "Large Language Models Meet NL2Code: A Survey",
       "authors": ["Daoguang Zan", "Bei Chen"],
       "year": 2023,
       "doi": "10.18653/v1/2023.acl-long.411",
       "relevance": "Survey of LLM code generation approaches, providing context for the evaluation methodology used."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "relevance": "Rigorous code generation evaluation methodology; the paper considers using LLMs for test case generation as future work based on this approach."
     },
     {
       "title": "Granite Code Models: A Family of Open Foundation Models for Code Intelligence",
       "authors": ["Mayank Mishra", "Matt Stallone"],
       "year": 2024,
       "arxiv_id": "2405.04324",
       "relevance": "Family of open code generation models relevant to the broader code LLM landscape evaluated."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Early work on LLM-based program synthesis with evaluation dataset design relevant to benchmark construction."
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "authors": ["Dan Hendrycks", "Steven Basart"],
       "year": 2021,
       "relevance": "Code generation benchmark providing principles for evaluation dataset design referenced by this work."
     }
   ]
 }