ai-research-survey

scan.json (32155B)

---

 {
   "paper": {
     "title": "From Evaluation to Enhancement: Large Language Models for Zero-Knowledge Proof Code Generation",
     "authors": [
       "Zhantong Xue",
       "Pingchuan Ma",
       "Zhaoyu Wang",
       "Yuguang Zhou",
       "Xiaoqin Zhang",
       "Shuai Wang",
       "Juergen Rahmel"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2509.11708",
     "doi": "10.48550/arXiv.2509.11708"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to a 'replication package' and 'Artifact' multiple times (§3.1.1, §13) but provides no repository URL or archive link in the paper text. 'Provided in the Artifact' is a reference to an intended release, not a working URL."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "ZK-Eval benchmark questions and filtered task IDs (Tables 9-10) are described, and the paper says the 'full set of questions is provided in the Artifact' (§3.1.1), but no URL to download the dataset is provided. They use public datasets (HumanEval, LiveCodeBench) but their contributed benchmark has no verifiable release link."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Linux (Ubuntu 22.04 LTS) server equipped with 256 GiB of RAM' (§5.1) but provides no requirements.txt, Dockerfile, library versions, or dependency specifications sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. §13 says prompts are 'available in the replication package' but no URL or README with commands is given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Tables 2-3, Figures 4-6, 9) report point estimates only. No confidence intervals, error bars, or ± notation appear despite using 10 samples per task to estimate Pass@1."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims (e.g., 'ZK-Coder delivers a significant improvement', 'GPT-o3 leads') based solely on comparing numbers. No statistical significance tests (p-values, t-tests, bootstrap tests) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports improvements with baseline context, e.g., 'improving code generation success rates from 20.29% to 87.85% on Circom and from 28.38% to 97.79% on Noir' (§1), and Table 2 provides both ZK-Coder and baseline accuracy for each model/language combination."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for benchmark sizes: 68 tasks from HumanEval, 34 from LiveCodeBench, 35 algebraic primitives, 172/164 MCQ questions. No power analysis or discussion of whether these sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite using 10 samples per task (§3.2), no standard deviations, interquartile ranges, or spread measures are reported. Tables 2-3 and all figures show single point estimates only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares ZK-Coder against a 'Simple Baseline' (direct generation with grammar summary and few-shot examples). Table 3 adds a 'Repairing Baseline' (baseline + interactive repair loop). §5.2 describes the baseline setup."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses GPT-o4-mini, GPT-o3, DeepSeek-V3, and Qwen3 — all contemporary models. Baselines use the same models with direct prompting, ensuring a fair comparison of the framework rather than the model."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "§5.3 provides a thorough ablation study with 6 variants (V1-V6): No RAG, No Sketch, No Compile-Repair, No Execute-Repair, Similarity-based RAG, Only Repair. Figure 9 shows results for all variants across both languages."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports: syntactic validity (compilable programs), semantic correctness (Pass@1 on test cases), per-stage metrics (sketch correctness, repair pass rate, program correctness), and average token cost (Table 2). MCQ accuracy is reported separately for language knowledge."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human experts validated the benchmark questions (§3.1.1: 'Three human reviewers independently assess each item'), but no human evaluation of ZK-Coder's generated code outputs is performed. All system evaluation is automated via test suites."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly uses hidden test cases: 'evaluation test cases remain hidden to prevent data-leakage' (§4.2.3). Table 2 measures 'Program correctness' as 'conditioned on all successfully repaired programs, and measures the proportion that pass all hidden test cases.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by: language (Circom/Noir), model (4 LLMs), pipeline stage (Table 2), primitive type and error category (Fig. 6), MCQ category (Fig. 4), ablation variant (Fig. 9), dataset type (Table 3), and failure mode (Table 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§5.5 (RQ7) is a dedicated failure analysis. Table 4 categorizes failures into 5 types: Repair Budget Exceed, Sketched Constraint Incorrect, False Acceptance, False Rejection, and Mixed. §3.4 also analyzes error categories in primitive implementations (Fig. 6)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that LLMs perform poorly on algebraic primitives (§3.4, even best category only 52%), end-to-end generation drops to 5-20% (§3.5), open-weight models achieve under 10% baseline accuracy on Circom, and the ablation shows significant degradation when components are removed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's key claims are supported: 'strong proficiency in language syntax' matches Finding ① (88.1% accuracy, §3.3), 'struggle when implementing and composing algebraic primitives' matches Findings ③-④ (§3.4), and '20.29% to 87.85% / 28.38% to 97.79%' matches Table 2 exactly."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claims are about ZK-Coder's components causing improvement. The ablation study (§5.3) uses controlled single-variable manipulation — each variant removes exactly one component — which adequately supports claims like 'removing either component causes a significant performance drop (over 10%)' (§1)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its scope to Circom and Noir (§2.1: 'In this work, we focus on Circom and Noir') and explains why. Claims are generally tied to specific languages and models. The title ('Zero-Knowledge Proof Code Generation') accurately reflects the scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§7 discusses five specific threats to validity: LLM-generated question bias (①), test case limitations (②-③), prompting strategy influence (④), and security limitations (⑤). These address specific alternative explanations for the observed results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between functional correctness (what it measures via test suites) and other ZK program qualities it does not claim: efficiency/circuit size (§7 'ZKP Program Efficiency'), security (§7 threat ⑤), and real-world deployment readiness. It also notes 'compilable alone is insufficient for assessing ZK program quality' (§3.5)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-o4-mini', 'GPT-o3', 'DeepSeek-V3', and 'Qwen3' (§3.2) without snapshot dates, API versions, or specific model IDs. These are marketing names that do not specify exact model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Fig. 7 shows high-level prompt sketches (e.g., '[Prompt] Given this problem, please formulate the verification constraints using ZKSL...') but not actual prompt text. §13 states 'All prompts used in our experiments are available in the replication package' but no URL is provided in the paper to access them."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Repair budgets are reported (N1=8, N2=3 in §4.2.3) and '10 samples per task' (§3.2), but LLM API parameters (temperature, top-p, max tokens) are not stated anywhere in the paper."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "ZK-Coder's agentic scaffolding is described in detail: §4.2 covers three stages (constraint formulation, sketch-guided retrieval, interactive refinement), Fig. 7 provides a workflow diagram, retry thresholds are specified (N1=8, N2=3), and the ZKSL grammar is formally defined in Fig. 8."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark curation pipeline is well-documented: §3.1.1 describes document collection (51 Circom docs, 40 Noir docs), LLM-aided question generation, and expert validation. §3.1.3 details task filtering criteria (excluding string/float/variable-length tasks), yielding 68 tasks. Tables 9-10 list exact task IDs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§7 'Discussion and Limitations' is a dedicated section covering ZKP program efficiency, LLM training/fine-tuning challenges, real-world representativeness, and five specific threats to validity (①-⑤)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§7 identifies five specific threats: ① LLM-aided question generation may introduce bias, ② hand-crafted test cases cannot prove correctness (mitigated by 86-87% mutation detection rates), ③ automated evaluation depends on test coverage, ④ prompting strategy may influence results, ⑤ generated code may have security vulnerabilities. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§7 explicitly states what the results do NOT show: efficiency/circuit size optimization is not addressed ('ZKP Program Efficiency' paragraph), fine-tuning is not explored ('LLM Training and Fine-tuning'), security guarantees are not provided (threat ⑤). §2.1 explicitly bounds scope to Circom and Noir, excluding zkVMs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Per-task results, individual model outputs, and raw Pass@1 data across 10 samples are not provided. Only aggregate statistics appear in the paper. The 'replication package' is mentioned but no URL is given."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§3.1 provides detailed benchmark construction: sources (official docs, GitHub repos, tutorials), LLM-aided question generation pipeline, algebraic primitive curation from circomlib/zk-kit/Z3Py (§3.1.2), HumanEval task adaptation with three specific transformations (§3.1.3), and quality validation through mutation testing."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Three expert reviewers validated the benchmark (§3.1.1) but are likely the authors, not recruited participants. The study evaluates LLMs on benchmarks, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw sources to final benchmark is documented: document collection → LLM question generation → expert validation → deduplication (§3.1.1); library survey → primitive extraction → completeness alignment (§3.1.2); HumanEval filtering → verification reformulation → oracle adaptation → test case generation (§3.1.3). Stage counts are provided (172/164 MCQ, 35 primitives, 68 tasks)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. No acknowledgments section listing grants or sponsors is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Hong Kong University of Science and Technology, Zhejiang University of Technology, CipherInsight Limited, and HSBC. The dual affiliations with CipherInsight Limited (a ZK-related company) are transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funder is disclosed. Two authors are affiliated with CipherInsight Limited (appears to be a ZK-related company) and one with HSBC. Without funding disclosure, independence cannot be assessed. The paper proposes ZK-Coder which could have commercial relevance to CipherInsight."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper. Two authors have dual affiliations with CipherInsight Limited (a company in the ZK space) but no declaration of financial interests is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four evaluated models (GPT-o4-mini, GPT-o3, DeepSeek-V3, Qwen3)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges HumanEval is 'potentially contaminated' (§5.4) and uses LiveCodeBench as a contamination-free alternative, but does not analyze train/test overlap for the adapted HumanEval benchmark or ZK-Eval MCQ questions, which are derived from public documentation that models likely trained on."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges HumanEval contamination risk and uses LiveCodeBench (§5.4) for RQ6, but the primary evaluation (Table 2, RQ4) uses adapted HumanEval without any contamination mitigation. The ZK-Eval MCQ benchmark is derived from public documentation, which could be in training data, with no contamination analysis."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates LLMs on benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports average token costs per task for each model/language/method combination. §5.2 states 'each task requiring fewer than 5,000 tokens on average, corresponding to less than 0.1 USD per task.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware ('Linux (Ubuntu 22.04 LTS) server equipped with 256 GiB of RAM' in §5.1) but does not state total API spend, total compute time, or aggregate cost for the full evaluation campaign."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses '10 samples per task to estimate Pass@1 rates and reduce variance' (§3.2) but does not report variance across these samples or any seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "§3.2 explicitly states '10 samples per task to estimate Pass@1 rates and reduce variance.' Repair budgets (N1=8, N2=3) are also stated in §4.2.3."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper uses 'standard zero-shot and few-shot prompting techniques without extensive prompt engineering' (§13) but does not document how repair budgets (N1=8, N2=3) or other parameters were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The repair budget thresholds (N1=8 for compile-repair, N2=3 for execute-repair) are stated but their selection is not justified. No exploration of alternative configurations is documented."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims across 4 models × 2 languages × multiple ablation variants but performs no statistical tests at all, and therefore no multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own ZK-Coder system against their own baseline implementations. No discussion of author-evaluation bias or independent evaluation. Per Lucic et al. (2018), authors' implementations of baselines systematically underperform."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Token costs are reported alongside accuracy (Table 2) but performance is not shown as a function of compute budget. The repair budgets are fixed (N1=8, N2=3) with no analysis of how performance varies with different compute allocations."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "§7 'Real-World Representativeness' analyzes construct validity: generated programs have average cyclomatic complexity of 22.81 vs 16.8/17.5 in production repos. §10 surveys 37 repositories (408 source files) showing 97.3% coverage for arithmetic and 94.6% for relational primitives. §3.1.3 justifies the HumanEval adaptation rationale."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "When comparing models (Table 2), all models use the same ZK-Coder pipeline, isolating the model effect. The ablation study (§5.3) systematically varies the scaffold components. Baseline comparisons use the same models with and without scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges HumanEval is 'potentially contaminated' and uses LiveCodeBench for RQ6 (§5.4), but does not discuss temporal leakage for the primary ZK-Eval benchmark or adapted HumanEval evaluation. No model training cutoff dates are provided to assess temporal overlap."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "§4.2.3 states 'evaluation test cases remain hidden to prevent data-leakage' and §13 verifies 'no test cases or ground-truth solutions were included in the prompts, preventing information leakage that could artificially inflate success rates.' The ZK-Coder self-generated test cases are separate from evaluation test cases."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data for the evaluated models contains ZK-related code from circomlib, zk-kit, or the GitHub repositories used to construct the benchmark. The 37 production repositories surveyed (§10) are public and could be in training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "LiveCodeBench is used as a temporal decontamination strategy for RQ6 only (§5.4). No concrete detection method (canary strings, membership inference, n-gram overlap) is applied to the primary evaluation benchmarks (ZK-Eval, adapted HumanEval)."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs demonstrate strong competence in ZK language knowledge, achieving accuracy within a few percentage points of human experts (88.1% for GPT-o4-mini vs 88.7% for human expert).",
       "evidence": "Fig. 4 shows MCQ benchmark accuracy across 4 categories for Circom and Noir. GPT-o4-mini reaches 88.1%, GPT-o3 87.2%, vs human expert 88.7% (§3.3).",
       "supported": "strong"
     },
     {
       "claim": "LLMs perform consistently weak on implementing and using algebraic primitives, with even the best category (logical operations) reaching only 52% accuracy in Circom and 49% in Noir.",
       "evidence": "Fig. 6 shows error distributions across primitive types and error categories. Comparison scored 37%/20%, arithmetic and composites extremely low (§3.4).",
       "supported": "strong"
     },
     {
       "claim": "ZK-Coder improves end-to-end code generation success rates from 20.29% to 87.85% on Circom and from 28.38% to 97.79% on Noir using GPT-o3.",
       "evidence": "Table 2 provides per-stage and overall accuracy with token costs. Per-stage breakdown: 97.30% sketch, 94.41% repair, 95.64% program correctness for GPT-o3 on Circom (§5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Each component of ZK-Coder (sketching, RAG, repair) is indispensable; removing any causes over 10% performance drop.",
       "evidence": "Fig. 9 shows ablation results: removing RAG drops Circom from 65.66% to 53.04%, removing sketch to 46.37%, removing compile-repair to 26.04% (§5.3).",
       "supported": "strong"
     },
     {
       "claim": "ZK-Coder generalizes to harder contamination-free tasks (LiveCodeBench) and production coding patterns.",
       "evidence": "Table 3: LiveCodeBench 44.10% Circom / 56.93% Noir vs Simple Baseline 1.72% / 9.89%. Production case studies: 90.83% / 92.09% (§5.4).",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning models (GPT-o4-mini, GPT-o3) substantially outperform open-weight models (DeepSeek-V3, Qwen3) on ZK code generation, by 2-3x on end-to-end tasks.",
       "evidence": "Fig. 5 and Table 2: GPT-o3 achieves 20%/28% on Circom/Noir baselines vs 5-7%/10-16% for open-weight models. With ZK-Coder, GPT-o3 at 87.85%/97.79% vs 42-50%/55-69% for open-weight (§3.5, §5.2).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ZK-Eval reveals that while LLMs achieve near-expert accuracy on ZK language knowledge (88%), they fail dramatically when implementing algebraic primitives (<52%) or generating end-to-end ZK programs (<30% baseline). ZK-Coder, an agentic framework combining constraint sketching, guided retrieval, and interactive repair, boosts GPT-o3's success rate from 20% to 88% on Circom and 28% to 98% on Noir. Ablation shows all three components are necessary, with compile-repair being the most critical (removing it drops accuracy to 26%). The dominant failure modes are repair budget exhaustion and incorrect constraint sketching.",
   "red_flags": [
     {
       "flag": "No error bars or statistical tests",
       "detail": "Despite 10 samples per task, no variance, confidence intervals, or significance tests are reported for any comparison. All claims of improvement are based on comparing point estimates. The paper makes dozens of comparative claims (4 models × 2 languages × 7 ablation variants) without any statistical rigor."
     },
     {
       "flag": "Primary benchmark contamination acknowledged but not mitigated",
       "detail": "The paper acknowledges HumanEval is 'potentially contaminated' (§5.4) but uses it for the primary evaluation (Table 2, RQ4). LiveCodeBench is used only for the generalization study (RQ6). The ZK-Eval MCQ benchmark is derived from public documentation that models likely trained on, with no contamination analysis."
     },
     {
       "flag": "Undisclosed potential commercial interest",
       "detail": "Two authors (Pingchuan Ma, Shuai Wang) are affiliated with CipherInsight Limited, which appears to be a company in the zero-knowledge proof space. One author (Juergen Rahmel) is from HSBC. No funding disclosure, competing interests statement, or financial interests declaration appears in the paper, despite the proposed ZK-Coder having potential commercial relevance."
     },
     {
       "flag": "Self-evaluation without bias acknowledgment",
       "detail": "The authors evaluate their own ZK-Coder system against their own baseline implementations. Per Lucic et al. (2018), authors' implementations of baselines systematically underperform. No discussion of this bias or independent evaluation is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark paper, the basis for ZK-Eval's end-to-end generation tasks and a core code generation evaluation resource."
     },
     {
       "title": "Livecodebench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code generation benchmark used for ZK-Coder's generalization study."
     },
     {
       "title": "A Survey on Large Language Models for Code Generation",
       "authors": ["Juyong Jiang", "Fan Wang", "Jiasi Shen"],
       "year": 2025,
       "doi": "10.1145/3747588",
       "relevance": "Comprehensive survey of LLM-based code generation frameworks and benchmarks."
     },
     {
       "title": "A Survey on Code Generation with LLM-based Agents",
       "authors": ["Yihong Dong", "Xue Jiang", "Jiaru Qian"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey of agentic LLM frameworks for code generation, the paradigm ZK-Coder follows."
     },
     {
       "title": "A survey on llm-based code generation for low-resource and domain-specific programming languages",
       "authors": ["Sathvik Joel", "Jie JW Wu", "Fatemeh H Fard"],
       "year": 2024,
       "arxiv_id": "2410.03981",
       "relevance": "Directly relevant survey on LLM code generation for DSLs and low-resource languages, the core challenge this paper addresses."
     },
     {
       "title": "MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "Multi-language code generation benchmark addressing evaluation across diverse programming languages."
     },
     {
       "title": "Knowledge transfer from high-resource to low-resource programming languages for code llms",
       "authors": ["Federico Cassano", "John Gouwar", "Francesca Lucchetti"],
       "year": 2024,
       "relevance": "Addresses knowledge transfer for low-resource language code generation, an alternative approach to ZK-Coder's RAG strategy."
     },
     {
       "title": "Enhancing code generation for low-resource languages: No silver bullet",
       "authors": ["Alessandro Giagnorio", "Alberto Martin-Lopez", "Gabriele Bavota"],
       "year": 2025,
       "arxiv_id": "2501.19085",
       "relevance": "Studies challenges of LLM code generation for low-resource languages, finding no single approach dominates."
     },
     {
       "title": "Retrieval-augmented generation for large language models: A survey",
       "authors": ["Yunfan Gao", "Yun Xiong", "Xinyu Gao"],
       "year": 2023,
       "arxiv_id": "2312.10997",
       "relevance": "Survey of RAG techniques that underpin ZK-Coder's sketch-guided retrieval component."
     },
     {
       "title": "ToolLLM: Facilitating large language models to master 16000+ real-world APIs",
       "authors": ["Yujia Qin", "Shihao Liang", "Yining Ye"],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Demonstrates LLM capabilities in API usage and tool integration, related to ZK-Coder's API-aware code generation."
     },
     {
       "title": "Automated detection of under-constrained circuits in zero-knowledge proofs",
       "authors": ["Shankara Pailoor", "Yanju Chen", "Franklyn Wang"],
       "year": 2023,
       "relevance": "Static analysis tool (QED2) for ZK circuit verification, complementary to ZK-Coder's generation approach."
     },
     {
       "title": "Grammar prompting for domain-specific language generation with large language models",
       "authors": ["Bailin Wang", "Zi Wang", "Xuezhi Wang"],
       "year": 2023,
       "relevance": "Grammar prompting technique for DSL generation compared against ZK-Coder's sketch-based approach in ablation."
     }
   ]
 }