ai-research-survey

scan.json (31518B)

---

 {
   "paper": {
     "title": "SolEval: Benchmarking Large Language Models for Repository-level Solidity Code Generation",
     "authors": [
       "Zhiyuan Peng",
       "Xin Yin",
       "Rui Qian",
       "Peiqin Lin",
       "Yongkang Liu",
       "Hao Zhang",
       "Chenhao Ying",
       "Yuan Luo"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2502.18793",
     "doi": "10.48550/arXiv.2502.18793"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SolEval is the first repository-level benchmark for Solidity smart contract generation, containing 1,125 filtered function samples from 9 real-world projects. Evaluation of 16 LLMs shows the best model (DeepSeek-V3) achieves only 26.29% Pass@10, revealing significant room for improvement. A correctness-gas efficiency trade-off was observed: models excelling at Pass@k generate gas-inefficient contracts. Supervised fine-tuning on Qwen-7B improved Pass@5 from 16.67% to 58.33%, demonstrating the benchmark's utility for model improvement.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'We release our data and code at https://github.com/pzy2000/SolEval.' A specific GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is released at the same GitHub URL. Appendix C.3 describes the dataset files (dataset.json, example.json, raw.json)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section B.2 describes hardware (Intel Xeon Gold 6226R, 192GB RAM, 8 NVIDIA RTX A8000 GPUs, Ubuntu 20.04.1 LTS) and mentions PyTorch and HuggingFace, but no requirements.txt, Dockerfile, or library version numbers are specified in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides some reproduction hints (fuzzing seed 666, approximate compute times: ~1 week for Table 4, ~24 hours for Table 5) but no step-by-step reproduction instructions. The GitHub repo is linked but the paper itself lacks sufficient detail to reproduce without consulting external resources."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 4, 5, 6, and 7 report only point estimates (e.g., '26.29%'). No confidence intervals, error bars, or ± notation appear despite results being averaged over five independent runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims like 'DeepSeek-V3 performs best' and 'RAG improves performance' are based solely on comparing raw numbers without any hypothesis tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The SFT experiment reports effect sizes with baseline context: 'Pass@5 increasing from 16.67% to 58.33%' (Table 6). The ablation study (Table 5) reports results for each configuration with comparable baselines, providing effect magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 1,125 (or 1,507) benchmark samples is sufficient. No power analysis is discussed. The sample size is a result of filtering criteria, not a deliberate design choice."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 states 'all experimental results are averaged over five independent runs' but no standard deviation, IQR, or spread measure is reported. Tables contain only point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "16 LLMs are compared against each other in Table 4, and the benchmark itself is compared against prior benchmarks (BenchSol, HumanEval, MBPP, etc.) in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include very recent releases: DeepSeek-V3, DeepSeek-R1, GPT-4o, QwQ-32B, and Qwen2.5-Coder, all from 2024-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 presents an ablation study on RAG and Context for DeepSeek-V3 with four conditions (±RAG × ±Context). Figure 4 studies the impact of different shot numbers on Qwen2.5-Coder-7B."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation metrics: Pass@k, Compile@k, Gas@k, Gas Fee, and Vul@k, covering functional correctness, compilation success, gas efficiency, and vulnerability rate."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation of LLM-generated code is entirely automated through test case execution, compilation checking, gas measurement via Forge, and vulnerability detection via Slither. Human annotators were involved in benchmark construction but not in evaluating model outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For SFT, Section 4.3 states: 'we ensured that there were no overlapping functions between the training and test sets' and used a separate set of repositories as the test set. For the main evaluation, the benchmark is a fixed test set not used for any model tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides per-model breakdowns across all metrics. Results are grouped by model size (6.7B-16B, 32B-671B). Table 2 provides per-project statistics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 and Figure 9 show a failure case where DeepSeek-R1-Distill-Qwen-7B fails an easy boolean AND operation. The paper discusses how models 'can fail in some really easy cases' while succeeding on harder problems."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that DeepSeek-R1-Distill-Qwen-7B, despite claiming o1-mini-level performance, underperforms CodeLlama-7B on Solidity. The correctness-gas trade-off is reported as a negative finding: the best-performing model (DeepSeek-V3) generates the most gas-inefficient contracts."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'best-performing LLM achieves only 26.29% Pass@10' matches DeepSeek-V3 in Table 4; 'Pass@5 increasing from 16.67% to 58.33%' matches Table 6; '1,507 samples from 28 different repositories' is stated (though internally inconsistent with the 1,125 filtered figure used in evaluation)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about RAG and Context improving performance are supported by a controlled ablation study (Table 5) with four conditions (±RAG × ±Context). The SFT improvement claim is supported by before/after comparison on the same model. These are adequate designs for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds claims to Solidity code generation specifically. The Limitations section explicitly notes the monolingual limitation, inability to test GPT-5/Claude, and metric limitations. The title and claims focus on Solidity specifically."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides single interpretations for findings without exploring alternatives. DeepSeek-R1-Distill's poor performance is attributed to 'lack of knowledge of Solidity' without considering other explanations (e.g., instruction-following vs coding, prompt sensitivity). No alternative explanations for the gas-correctness trade-off are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@k, Compile@k, Gas@k, and Vul@k, and frames these as measuring code generation correctness, compilability, gas efficiency, and security. The metrics directly measure the claimed outcomes. The paper also discusses the distinction between functional correctness and broader quality (gas, vulnerability)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models are identified by name and size (e.g., 'DeepSeek-Coder-6.7B', 'CodeLlama-7B') but without exact checkpoint/HuggingFace IDs. Closed-source models use marketing names: 'GPT-4o' and 'GPT-4o-mini' without snapshot dates or API versions. 'DeepSeek-V3' is accessed via API without version specification."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates with actual text are provided in Figures 5-6 and Appendix C. The prompts include exact markers ('// IMPLEMENT THE FUNCTIONALITY BASED ON THE PROVIDED REQUIREMENT', '// START_OF_REQUIREMENT', etc.) and complete examples."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports: n=10 samples, nucleus sampling with temperature=1, top-p=0.95, max generation length=512, greedy search for k=1. Section 4.3 reports SFT hyperparameters: max input length 2048, 3 epochs, 9:1 train/validation split, TRL library defaults."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is direct LLM inference with few-shot prompting. RAG is used for example selection but is not agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 and Figure 3 document the full pipeline: project selection (64 projects from 6 GitHub organizations), function parsing (17,823 → 1,125 after filtering), test construction, human annotation by 5 annotators, and context parsing. Filtering criteria are explicit (LOC ≥ 5, exclude test/deprecated functions, etc.)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section lists four specific limitations: monolingual benchmark, inability to evaluate GPT-5/Claude Opus, metrics don't provide optimization mechanisms, and scope limited to tested models."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: monolingual focus on Solidity excluding Vyper and Rust, funding constraints preventing GPT-5/Claude evaluation, gas/vulnerability metrics evaluate but don't optimize. These are concrete, not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what was NOT tested: other smart contract languages (Vyper, Rust), newer models (GPT-5, Claude Opus), and optimization/remediation methods. Future work directions explicitly bound the current scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is released at https://github.com/pzy2000/SolEval. Appendix C.3 describes the data files: dataset.json (source information), example.json (RAG examples), and raw.json."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full data collection process in five phases: project selection from 6 GitHub organizations, function parsing with Tree-sitter, test construction with Forge, human annotation with inter-annotator agreement (Fleiss' κ = 0.8), and context parsing via static analysis."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "For annotators: Section 3.4 and Appendix E describe recruiting 'five master's students with at least three years of Solidity experience' compensated at $100/hour. For benchmark data: Section 3.1 describes selecting from 'six popular GitHub organizations' sorted by star count, filtering low-star projects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 and Section 3 document the full pipeline with counts at each stage: 64 projects → 17,823 functions → 1,125 functions from 9 projects after filtering → 2,217 test cases → human annotation → context parsing → final SolEval benchmark."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists: National Key Research and Development Program of China (Grant 2024YFB2705300), Shanghai Science and Technology Innovation Action Plan (Grant 23511100400), National Natural Science Foundation of China (Grant 62402313), and Open Research Fund of the State Key Laboratory of Blockchain and Data Security."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Shanghai Jiao Tong University, Shanghai SimMed Technology Co, Zhejiang University, Fudan University, LMU Munich, Munich Center for Machine Learning, Northeastern University, and Universiti Sains Malaysia. Authors are not affiliated with any of the evaluated model providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from Chinese government research programs (National Key R&D, NSFC, Shanghai Action Plan) and a university state key laboratory. None of the funders have a financial interest in how specific LLMs perform on the benchmark."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. One author is affiliated with 'Shanghai SimMed Technology Co, Ltd' but no competing interests statement addresses whether this creates a conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 16 evaluated models. The paper cites model papers but does not specify when their training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses human-annotated comments to reduce memorization of prompts (Section 3.4), but does not discuss whether the code functions themselves (the expected outputs from popular repos like OpenZeppelin) overlap with model training data. The code-level overlap is the primary contamination concern and is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark code comes from popular, high-star GitHub repositories (OpenZeppelin, Solady, etc.) that are almost certainly in the training data of models like GPT-4o and DeepSeek-V3. The paper addresses prompt memorization through human annotation but does not address whether models have memorized the ground-truth code functions."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates LLMs on a benchmark, not human participants. Human annotators constructed the benchmark but are not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied. The benchmark evaluates LLMs, not humans."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation. Annotator details (master's students with 3+ years Solidity experience) are for benchmark construction, not as study subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied in the evaluation."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions assigned to humans."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 reports average runtime per example (~1.6s to ~2.8s depending on shot count) and average token length (733-1706 tokens). Section B.2 provides total reproduction time estimates: 'approximately one week' for Table 4, '24 hours' for Table 5."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section B.2 states the hardware: '16-core workstation equipped with an Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz, 192GB RAM, and 8 NVIDIA RTX A8000 GPUs.' Approximate wall-clock time is given for each experiment. The section notes 'The computational budget, including GPU hours, the number of GPUs, and the total parallelism across them, is crucial.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 states 'all experimental results are averaged over five independent runs' but no per-run results, standard deviation, or seed sensitivity analysis is reported. Only averaged point estimates appear in the tables."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 explicitly states: 'we set the total number (denoted as n) of samples generated by an LLM to 10' and 'all experimental results are averaged over five independent runs.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. For SFT, Section 4.3 states 'All other hyperparameters were kept at the default values provided by the TRL library' suggesting no search was conducted. For inference, fixed hyperparameters are used without mentioning alternatives tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The 1-shot setting is justified via Figure 4, which shows diminishing returns with more shots alongside sharply rising cost. Qwen-7B is chosen for SFT 'due to its strong performance in initial evaluations.' All model configurations are reported transparently."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed in the paper, so correction for multiple comparisons is structurally inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors create the benchmark and evaluate models on it without acknowledging potential biases in benchmark construction choices (e.g., function selection, test case design) that could favor certain models or coding patterns."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 6.7B to 671B parameters are compared without normalizing for compute. DeepSeek-V3 (671B) is compared directly against 7B models without discussing the 100x compute difference. No performance-per-compute analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper argues for repository-level evaluation over standalone functions but does not discuss whether Pass@k on their specific 1,125 functions (from 9 projects, mostly from a few large repos like Solady) actually measures general Solidity code generation capability. No discussion of construct validity or comparison with alternative evaluation approaches."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. All models are evaluated with the same direct prompting setup (same prompt template, same RAG/context configuration)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark code comes from established GitHub repos (OpenZeppelin, Solady, etc.) that predate most evaluated models' training data. The paper does not discuss temporal leakage — whether the ground-truth code existed before the models' training cutoffs."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (providing function signatures, requirements, and repository context) leaks information beyond what a developer would have in real-world code generation scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "For SFT, Section 4.3 ensures no overlapping functions and uses repository-level splits. However, for the main 16-model evaluation, there is no discussion of whether the benchmark functions from popular repos are independent of model training data. The code from OpenZeppelin et al. is almost certainly in training sets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 uses human-annotated comments as a concrete prevention method: 'to reduce the LLMs' memorization effects, as original comments are highly likely to have been encountered during the pre-training phase.' For SFT, Section 4.3 applies function-level deduplication and excludes 9/30 repos with 'potential data leakage.'"
       }
     }
   },
   "claims": [
     {
       "claim": "The best-performing LLM (DeepSeek-V3) achieves only 26.29% Pass@10 on SolEval, indicating substantial room for improvement in Solidity code generation.",
       "evidence": "Table 4 shows DeepSeek-V3 at 26.29% Pass@10, with all other models scoring between 5.91% and 23.70%.",
       "supported": "strong"
     },
     {
       "claim": "Supervised fine-tuning on Qwen-7B using SolEval improves Pass@5 from 16.67% to 58.33%.",
       "evidence": "Table 6 reports before/after SFT metrics: Pass@5 16.67%→58.83%, Compile@5 66.67%→100%, Gas@1 0%→19.84%, Vul@1 26.61%→7.35%.",
       "supported": "moderate"
     },
     {
       "claim": "There is a fundamental trade-off between functional correctness and gas efficiency in Solidity code generation: DeepSeek-V3 ranks highest in Pass@10 but generates contracts with high gas fees.",
       "evidence": "Table 4/7 shows DeepSeek-V3 highest in Pass@k but with gas fee of -7525, while smaller models like DeepSeek-R1-Distill-Qwen-7B (lowest Pass@10 at 5.91%) generates cheaper contracts (fee -3472).",
       "supported": "moderate"
     },
     {
       "claim": "RAG and context information improve LLMs' performance in Solidity code generation.",
       "evidence": "Table 5 ablation on DeepSeek-V3: Pass@1 increases from 20.17% (no RAG, no context) to 21.72% (both RAG and context). Compile@1 increases from 50.32% to 53.35%.",
       "supported": "weak"
     },
     {
       "claim": "DeepSeek-R1-Distill-Qwen-7B underperforms CodeLlama-7B on Solidity despite claiming comparable performance to ChatGPT-o1-mini on mainstream benchmarks.",
       "evidence": "Table 4: DeepSeek-R1-Distill-Qwen-7B achieves 5.91% Pass@10 vs CodeLlama-7B at 14.26%.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Benchmark contamination from popular repositories",
       "detail": "The benchmark uses code from highly popular GitHub repositories (OpenZeppelin, Solady, Foundry) that are almost certainly in the training data of all evaluated models. The paper addresses prompt memorization through human annotation but does not address whether models have memorized the ground-truth code functions themselves."
     },
     {
       "flag": "No error bars despite multiple runs",
       "detail": "Results are averaged over five independent runs but no standard deviation, confidence intervals, or variance measures are reported. The reader cannot assess result stability or determine whether performance differences between models are statistically meaningful."
     },
     {
       "flag": "Internal inconsistency in sample counts",
       "detail": "The abstract and Table 1 claim 1,507 samples from 28 repositories, but Table 2 shows 1,125 filtered functions from 9 projects, and Figure 2 states 'SolEval benchmark is made up of 1,125 samples.' The discrepancy is not explained."
     },
     {
       "flag": "Small effect sizes for RAG/Context with no significance tests",
       "detail": "The ablation study (Table 5) shows RAG improves Pass@1 by only ~1.5 percentage points (20.17%→21.72%), which could easily be within noise given no statistical tests or error bars are provided. The paper presents this as a meaningful improvement."
     },
     {
       "flag": "SFT evaluation details incomplete",
       "detail": "The SFT evaluation test set composition (21 repos after excluding 9 of 30 for data leakage) is not fully described: how many test functions, from which domains, and whether the test repos are representative. The dramatic improvement (16.67%→58.33%) could partly reflect overfitting or data characteristics."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval, the foundational benchmark for LLM code generation evaluation that SolEval extends to Solidity."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["Hao Yu", "Bo Shen", "Dezhi Ran"],
       "year": 2024,
       "relevance": "Repository-level code generation benchmark for Python and Java that directly inspires SolEval's approach to non-standalone functions."
     },
     {
       "title": "DevEval: Evaluating code generation in practical software projects",
       "authors": ["Jia Li", "Ge Li", "Yunfei Zhao"],
       "year": 2024,
       "arxiv_id": "2401.06401",
       "relevance": "Large-scale repository-level code generation benchmark for Python with 1,825 samples, a direct comparison point for SolEval."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Class-level code generation benchmark evaluating LLMs beyond standalone functions."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code generation model family evaluated in SolEval, serving as a baseline for Solidity code generation."
     },
     {
       "title": "MultiPL-E: A scalable and polyglot approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2023,
       "relevance": "First multi-language parallel code generation benchmark extending HumanEval to 18 languages, highlighting the gap in Solidity support."
     },
     {
       "title": "Benchmarking large language models for ethereum smart contract development",
       "authors": ["Etienne Daspe", "Mathis Durand", "Julien Hatin"],
       "year": 2024,
       "relevance": "BenchSol: the only prior Solidity code generation benchmark (15 samples), which SolEval aims to supersede."
     },
     {
       "title": "DeepSeek-R1",
       "authors": ["DeepSeek"],
       "year": 2025,
       "relevance": "Reasoning-focused LLM whose distilled versions are evaluated and found to underperform on Solidity despite strong mainstream benchmark scores."
     },
     {
       "title": "Magicoder: Source code is all you need",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu"],
       "year": 2023,
       "arxiv_id": "2312.02120",
       "relevance": "Open-source code generation model using synthetic data, evaluated as a baseline in SolEval."
     },
     {
       "title": "OpenCodeInterpreter: Integrating code generation with execution and refinement",
       "authors": ["Tianyu Zheng", "Ge Zhang", "Tianhao Shen"],
       "year": 2024,
       "arxiv_id": "2402.14658",
       "relevance": "Code generation system with execution-based refinement, evaluated as a baseline for Solidity code generation."
     },
     {
       "title": "A3-CodGen: A repository-level code generation framework for code reuse with local-aware, global-aware, and third-party-library-aware",
       "authors": ["Dianshu Liao", "Shidong Pan", "Xiaoyu Sun"],
       "year": 2024,
       "relevance": "Repository-level code generation framework addressing context dependencies, directly relevant to SolEval's approach to non-standalone functions."
     },
     {
       "title": "DS-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Yuhang Lai", "Chengxi Li", "Yiming Wang"],
       "year": 2023,
       "relevance": "Data science code generation benchmark introducing non-standalone functions and addressing data leakage through function modification."
     }
   ]
 }