ai-research-survey

scan.json (29032B)

---

 {
   "paper": {
     "title": "FVEval: Understanding Language Model Capabilities in Formal Verification of Digital Hardware",
     "authors": [
       "Minwoo Kang",
       "Mingjie Liu",
       "Ghaith Bany Hamad",
       "Syed Suhaib",
       "Haoxing Ren"
     ],
     "year": 2024,
     "venue": "Design, Automation and Test in Europe (DATE)",
     "arxiv_id": "2410.23299",
     "doi": "10.23919/DATE64628.2025.10992720"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'Our benchmark and evaluation code is available at https://github.com/NVlabs/FVEval.' A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark test instances (NL2SVA-Human, NL2SVA-Machine, Design2SVA) are released as part of the GitHub repository referenced in the abstract."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using vLLM for open-source model inference but provides no requirements.txt, Dockerfile, or detailed environment specification listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Furthermore, full reproduction requires commercial FV tools (Cadence Jasper) for functional correctness evaluation, which are not freely available."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, 3, 4, 5 report point estimates only. No confidence intervals, error bars, or ± notation is used."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Model comparisons are based solely on comparing raw accuracy numbers without any hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw accuracy values (e.g., 0.456, 0.253). The paper mentions 'an improvement of up to 11%' once (Section 4.2) but does not systematically report effect sizes with baseline context across comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes: NL2SVA-Human has 79 assertions, NL2SVA-Machine has 300 test cases, Design2SVA has 96 per category. No power analysis or rationale for these numbers."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported. Greedy decoding results are single-run. Pass@k results use the unbiased estimator from Chen et al. but no spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple models are compared against each other: GPT-4o, Gemini-1.5-pro/flash, Mixtral-8x22b, Llama-3/3.1 8B and 70B variants, providing mutual baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include gpt-4o (May 2024), Gemini-1.5-pro/flash, Llama 3.1, all contemporary state-of-the-art models at the time of evaluation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies evaluation settings: zero-shot vs 3-shot prompting (Tables 1 vs 3), greedy decoding vs nucleus sampling, and pass@1 vs pass@3 vs pass@5 (Tables 2, 4, 5), showing the effect of each factor."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation metrics are used: syntax correctness, functional correctness (exact equivalence), partial functional correctness (implication relationship), and BLEU score."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated using commercial FV tools for syntax and formal equivalence checking. No systematic human evaluation of model outputs is performed, though qualitative failure examples are discussed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark test instances are separate from any training or ICL data. The 3 hand-crafted in-context examples for NL2SVA-Machine (Figure 15) are fixed and distinct from the 300 test cases."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by sub-benchmark (NL2SVA-Human, NL2SVA-Machine, Design2SVA) and within Design2SVA by design category (Pipeline vs FSM) in Table 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Detailed failure case analysis is provided in Figures 7 (NL2SVA-Human), 8 (NL2SVA-Machine), and 9 (Design2SVA), with discussion of common failure modes including SVA syntax hallucination and incorrect temporal logic formulation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings: BLEU scores do not correlate with functional correctness (Figure 6, correlation 0.056 and 0.093); in-context examples can serve as 'distractors' for smaller models (Section 4.3, Figure 8); functional correctness remains low even for best models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims FVEval is 'the first comprehensive benchmark' for LLM evaluation in FV (supported by the literature review in Section 5) and that 'a wide range of existing LLMs' are evaluated (8 models across proprietary and open-source, confirmed in Section 4.1)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes limited causal claims. The claim that in-context examples 'can aid models' (Section 4.3) is supported by controlled comparisons (same model, same test set, zero-shot vs 3-shot). The claim that sampling improves accuracy is supported by pass@1 vs pass@k comparisons."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to SystemVerilog assertions and hardware formal verification. The title specifies 'Formal Verification of Digital Hardware.' Section 6 explicitly notes limitations to current design types and evaluation methods."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for observed performance differences between models. No consideration of whether training data composition, architecture differences, or other confounds could explain the results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures SVA syntax correctness and formal functional equivalence, which directly correspond to the claimed capability (generating correct formal assertions). Section 3 explicitly describes what each sub-benchmark measures and why. The proxy closely matches the claim."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 specifies exact versions: 'gpt-4o-0513', 'gemini-1.5-pro-001', 'gemini-1.5-flash-001', 'mixtral-8x22b-instruct-v1', and specific Llama model sizes (8B, 70B) with family versions (3 and 3.1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in the appendix: Figure 12 (NL2SVA-Human), Figure 14 (NL2SVA-Machine), Figure 15 (ICL examples), and Figure 21 (Design2SVA), including system and user messages."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Hyperparameters are reported: greedy decoding for main experiments, T=1.0 for NL description generation, T=0.0 for critic assessment (Section 3.3), nucleus sampling with top_p=0.95 and t=0.8 for pass@k experiments (Section 4.4)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. All evaluations use direct prompting of LLMs without tool use, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data generation processes are well-documented: NL2SVA-Machine's 4-step pipeline (random SVA generation → LLM NL description → LLM critic assessment → human inspection, Section 3.3), Design2SVA's parameterized RTL generation with control parameters (Section 3.4, Figure 4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations and Future Work' and provides substantive discussion of the benchmark's current constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 identifies specific limitations: only arithmetic pipelines and FSMs as synthetic designs, no exploration of prompting schemes or structural inference methods, no tool-feedback or symbolic reasoning integration. These are specific to this study's design choices."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what was not tested: different styles of design modules, advanced prompting techniques (citing [47]), structural inference methods (citing [51, 50]), and LLM-agentic frameworks with tool feedback."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (test instances and evaluation code) is released at https://github.com/NVlabs/FVEval, enabling independent verification of the benchmark composition."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "NL2SVA-Human: expert-written testbenches covering FIFOs, arbiters, counters, RAM, FSMs (Table 6, 79 assertions). NL2SVA-Machine: 4-step automated generation (Section 3.3). Design2SVA: parameterized synthetic RTL with control parameters (Section 3.4, Figure 4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied. The benchmark data consists of expert-written verification collateral and synthetically generated test cases, not human subject data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The NL2SVA-Machine pipeline documents each step: (1) random SVA generation, (2) LLM NL description generation with T=1.0, (3) LLM critic assessment with T=0.0, (4) human inspection (Section 3.3). Design2SVA documents control parameters and sweep methodology (Section 3.4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources (grants, corporate sponsors) are disclosed. The Acknowledgements thank individuals and Cadence Design Systems but do not mention specific funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: UC Berkeley (first author) and NVIDIA (four co-authors). The first author's internship at NVIDIA is noted."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Four of five authors are from NVIDIA, a chip design company with direct commercial interest in LLM-driven FV automation. While the paper is a benchmark rather than a product evaluation, NVIDIA benefits from advancing this research area."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states proprietary models were accessed in May 2024 (Section 4.1) but does not state the training data cutoff dates for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether NL2SVA-Human expert-written testbenches or their underlying design patterns could appear in LLM training data. The synthetic nature of NL2SVA-Machine and Design2SVA mitigates this concern but is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No contamination analysis is performed. While synthetic test case generation (NL2SVA-Machine, Design2SVA) inherently reduces contamination risk, the paper does not frame this as a contamination mitigation strategy or address contamination for the expert-written NL2SVA-Human subset."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied. The paper evaluates LLMs on benchmark tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported despite evaluating 8 models across three benchmarks with multiple sampling configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper mentions using vLLM for open-source model inference but does not report GPU hours, hardware specifications, or total API costs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is performed. Greedy decoding results are single-run deterministic. Pass@k experiments use nucleus sampling but do not report sensitivity to random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total number of samples generated for pass@k computation is not explicitly stated. The paper references the unbiased estimator from Chen et al. [6] but does not specify n (total samples per problem)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Temperature and sampling parameters appear to use standard values without systematic tuning, but no search process or budget is documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results for all evaluated configurations (zero-shot, 3-shot, greedy decoding, nucleus sampling) rather than selecting only the best. Tables 1 and 3 show zero-shot and 3-shot side by side."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not acknowledge potential bias from evaluating third-party models on their own benchmark. As benchmark creators, design choices could inadvertently favor or disfavor certain model architectures."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-compute analysis is provided despite comparing models of vastly different sizes (8B to 70B+ parameters) and architectures (dense vs MoE)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 explicitly discusses what capabilities each sub-benchmark measures and their relevance to industrial FV workflows. Figure 6 demonstrates that BLEU is insufficient for evaluating SVA correctness, validating the need for formal equivalence checking."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models are evaluated via direct prompting without agentic frameworks or tool augmentation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The NL2SVA-Human expert-written testbenches and common design patterns (FIFOs, arbiters) may exist in model training data. Synthetic generation for NL2SVA-Machine and Design2SVA mitigates this but is not discussed as a leakage prevention measure."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setup or prompt format could leak information about expected answers."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether test cases share structural similarities that could inflate performance estimates (e.g., FIFO variants sharing common assertion patterns)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. The synthetic generation approach inherently reduces contamination risk but is not framed or analyzed as such."
       }
     }
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "claims": [
     {
       "claim": "FVEval is the first comprehensive benchmark for evaluating LLMs on tasks pertaining to hardware formal verification via SystemVerilog assertions.",
       "evidence": "Section 5 (Related Work) reviews prior work and notes existing evaluations are 'limited to less than ~20 cases of test instances and have considered a limited variety of task settings,' while FVEval covers 79 + 300 + 192 test instances across three sub-tasks.",
       "supported": "strong"
     },
     {
       "claim": "Proprietary models with advanced code generation capabilities (gpt-4o, Gemini) achieve the best performance on NL2SVA-Human across all metrics.",
       "evidence": "Table 1 shows gpt-4o achieves highest functional correctness (0.456) and partial functional correctness (0.582). Gemini-1.5-flash achieves highest syntax (0.949).",
       "supported": "strong"
     },
     {
       "claim": "BLEU scores do not correlate with formal functional correctness for SVA assertions.",
       "evidence": "Figure 6 shows correlation coefficients of 0.056 (GPT-4o) and 0.093 (Llama-3.1-70B) between BLEU scores and functional correctness.",
       "supported": "strong"
     },
     {
       "claim": "Sampling multiple response candidates improves accuracy, with state-of-the-art models achieving up to 11% improvement from pass@1 to pass@5.",
       "evidence": "Table 2 shows gpt-4o partial functional pass@5 (0.651) vs pass@1 (0.582 from Table 1), approximately a 7 percentage point improvement. Section 4.2 claims 'up to 11%' improvement.",
       "supported": "moderate"
     },
     {
       "claim": "In-context examples can improve functional correctness but may also serve as distractors for smaller models.",
       "evidence": "Table 3 shows improvement for gpt-4o (0.430 to 0.467 functional) and Llama-3.1-70b (0.303 to 0.457) with 3-shot, but Llama-3.1-8b degrades (0.320 to 0.267). Figure 8 provides qualitative examples.",
       "supported": "strong"
     },
     {
       "claim": "LLMs can identify and generate provable SVA assertions directly from design RTL without human guidance.",
       "evidence": "Table 5 shows pass@5 functional correctness of 0.500 (gemini-1.5-pro, pipeline) and 0.900 (gpt-4o, FSM). Figure 9 shows qualitative examples of proven assertions.",
       "supported": "strong"
     },
     {
       "claim": "All models achieve near-perfect pass@5 syntax correctness on Design2SVA (1.000 for all models).",
       "evidence": "Table 5 shows Syntax@5 = 1.000 for all six models on both pipeline and FSM test categories.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "FVEval is the first comprehensive benchmark for evaluating LLMs on hardware formal verification, comprising three sub-tasks (NL2SVA-Human, NL2SVA-Machine, Design2SVA) with 571 total test instances. State-of-the-art proprietary models achieve at best ~46% functional correctness on assertion generation from natural language and up to 90% pass@5 on generating provable assertions from FSM design RTL, while BLEU scores show near-zero correlation with actual functional correctness. In-context examples and multiple sampling improve performance but cannot close the gap, with smaller models sometimes degraded by fixed in-context examples.",
   "red_flags": [
     {
       "flag": "NVIDIA conflict of interest",
       "detail": "Four of five authors are from NVIDIA, a company with direct commercial interest in LLM-driven hardware verification automation. The benchmark could shape the field's evaluation criteria in ways favorable to NVIDIA's research priorities. Additionally, Cadence Design Systems contributed to the project (Acknowledgements) while their commercial tool is required for evaluation."
     },
     {
       "flag": "Commercial tool dependency limits reproducibility",
       "detail": "Functional correctness evaluation requires Cadence Jasper, a commercial formal verification tool. Independent researchers without access to this tool cannot fully reproduce the evaluation, limiting the benchmark's accessibility despite code release."
     },
     {
       "flag": "No contamination analysis",
       "detail": "No analysis of whether test cases (particularly the 79 expert-written NL2SVA-Human instances based on common design patterns like FIFOs and arbiters) appear in LLM training data. Common hardware verification patterns are widely available in textbooks and online resources."
     },
     {
       "flag": "No statistical rigor in model comparisons",
       "detail": "All model comparisons are based on raw point estimates without confidence intervals, error bars, or significance tests. Performance differences of a few percentage points are discussed as meaningful without any statistical validation."
     },
     {
       "flag": "Small benchmark for NL2SVA-Human",
       "detail": "NL2SVA-Human contains only 79 test instances across 13 testbenches and 6 module types. This small sample size limits the reliability of performance estimates and makes it difficult to draw robust conclusions about model capabilities on expert-written specifications."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 Technical Report",
       "authors": ["Josh Achiam et al."],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM evaluated in the benchmark; relevant to survey scope of LLM capabilities."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen et al."],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and pass@k metric used in this work; foundational code generation benchmark."
     },
     {
       "title": "ChipNeMo: Domain-Adapted LLMs for Chip Design",
       "authors": ["Mingjie Liu et al."],
       "year": 2023,
       "arxiv_id": "2311.00176",
       "relevance": "Domain-adapted LLMs for chip design including verification; directly relevant to AI-assisted code generation in specialized domains."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez et al."],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for LLM software engineering capabilities; methodological parallel to FVEval's benchmark design approach."
     },
     {
       "title": "VerilogEval: Evaluating Large Language Models for Verilog Code Generation",
       "authors": ["Mingjie Liu et al."],
       "year": 2023,
       "relevance": "Closest prior benchmark for LLM hardware code generation; FVEval extends this to formal verification specifically."
     },
     {
       "title": "VeriGen: A Large Language Model for Verilog Code Generation",
       "authors": ["Shailja Thakur et al."],
       "year": 2023,
       "arxiv_id": "2308.00708",
       "relevance": "Fine-tuned LLMs for Verilog generation; relevant to AI-assisted hardware code generation."
     },
     {
       "title": "AssertLLM: Generating and Evaluating Hardware Verification Assertions from Design Specifications via Multi-LLMs",
       "authors": ["Wenji Fang et al."],
       "year": 2024,
       "arxiv_id": "2402.00386",
       "relevance": "Prior work on LLM-based formal assertion generation from specifications; direct predecessor to FVEval."
     },
     {
       "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei et al."],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique; cited as potential improvement avenue for LLM formal verification capabilities."
     },
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters",
       "authors": ["Charlie Snell et al."],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Cited as evidence that inference-time compute scaling could improve LLM assertion generation, relevant to AI capability research."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["Shunyu Yao et al."],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Structural inference method cited as future improvement direction for LLM-based formal verification."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao et al."],
       "year": 2023,
       "relevance": "Agentic reasoning framework cited as potential approach for LLM-driven formal verification workflows."
     },
     {
       "title": "Mixtral of Experts",
       "authors": ["Albert Q. Jiang et al."],
       "year": 2024,
       "arxiv_id": "2401.04088",
       "relevance": "Sparse MoE architecture evaluated in the benchmark; relevant to understanding model architecture effects on code generation."
     }
   ]
 }