ai-research-survey

scan.json (28999B)

---

 {
   "paper": {
     "title": "Qiskit HumanEval: An Evaluation Benchmark For Quantum Code Generative Models",
     "authors": [
       "Sanjay Vishwakarma",
       "Francis Harkins",
       "Siddharth Golecha",
       "Vishal Sharathchandra Bajpe",
       "Nicolas Dupuis",
       "Luca Buratti",
       "David Kremer",
       "Ismael Faro",
       "Ruchir Puri",
       "Juan Cruz-Benito"
     ],
     "year": 2024,
     "venue": "International Conference on Quantum Computing and Engineering (QCE)",
     "arxiv_id": "2406.14712",
     "doi": "10.1109/QCE60285.2024.00137"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces Qiskit HumanEval (QHE), a 101-task benchmark for evaluating LLM quantum code generation using Qiskit. IBM's Granite-8B-Code-QK model, fine-tuned on Qiskit-specific data, achieves 46.53% pass@1 on QHE, outperforming all baselines including models 4x larger, with a 17.8 percentage point gain over the un-tuned Granite base model. No model passes either of the 2 difficult-level tasks. Fine-tuning for Qiskit slightly degrades general HumanEval performance (39.02% → 38.41%).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the dataset will be made publicly available as open source in Q3 2024' and 'The complete dataset is scheduled for release in the upcoming months.' This is a promise of future release, not a working link or archive at publication time."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same as above — the QHE dataset was not released at time of publication. Section V states 'The preliminary public release will offer a set of 150 tests' in a future release. No download link or repository URL is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'compatible with Qiskit ≥1.0' and the base model is on HuggingFace, but no requirements.txt, Dockerfile, or detailed dependency list is provided. Library versions beyond Qiskit are not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the evaluation approach conceptually but does not provide scripts, commands, or a README for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables II and III report only point estimates (e.g., '46.53%') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'Our Qiskit model GRANITE-8B-CODE-QK performs better than all the other baselines on QHE' based solely on comparing raw pass@1 numbers with no statistical significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 'The pass rate jumps 17.8 points compared with GRANITE-8B-CODE-BASE' with both baseline (28.71%) and tuned (46.53%) values, providing sufficient context for the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 101 tasks with no justification for this sample size. The difficulty distribution (54 basic, 45 intermediate, 2 difficult) is described but not justified, and the 2-task difficult category is extremely small."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are from greedy decoding (single deterministic pass). No variance across runs, seeds, or sampling strategies is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table II includes 5 baseline models: CodeLlama-34B-Python-hf, DeepSeek-Coder-33B-base, StarCoder2-15B, CodeGemma-7B, and Granite-8B-Code-Base."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines (CodeLlama, DeepSeek-Coder, StarCoder2, CodeGemma, Granite) are all from the 2023-2024 era, which is contemporary for a mid-2024 paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The fine-tuning pipeline involves multiple stages (extended pre-training on Qiskit data, then instruction tuning with multiple data sources) but no ablation isolates the contribution of each component or data source."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Pass@1 with greedy decoding is reported. No Pass@5, Pass@10, or other metrics (code quality, efficiency, etc.) are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via unit test pass/fail. No human evaluation of generated code quality, readability, or correctness beyond test passing."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The QHE dataset is described as 'hand-curated' and 'not part of the training dataset for the large language models being tested.' Tasks are stated to be 'original and not extracted from any existing tutorials, courses, books, or papers.' The benchmark functions as a held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table III provides per-difficulty breakdowns (basic, intermediate, difficult) for all models. Table I also categorizes the 101 tasks into 8 categories, though results are not broken down by these categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes 'No model is currently able to pass either of the 2 difficult tests' but provides no analysis of why models fail, what types of errors occur, or which categories are hardest."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table II shows that Granite-8B-Code-QK's general HumanEval score slightly decreases from 39.02% to 38.41% after Qiskit fine-tuning, and all models fail both difficult tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the work 'demonstrate[s] the feasibility of using LLMs for generating quantum code' and 'establish[es] a new benchmark.' The results (models achieving 24-46% pass rates) support feasibility, and the benchmark is indeed introduced."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that Qiskit-specific fine-tuning improves QHE performance ('The pass rate jumps 17.8 points... which shows the efficiency of our tuning approach'). The comparison of Granite-8B base vs. Granite-8B-QK (same architecture, different training) is a reasonable controlled comparison adequate for this claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is titled specifically for 'Quantum Code Generative Models' and scoped to the Qiskit SDK. Claims stay within the tested domain. Section V discusses Qiskit-specific limitations and future expansion to OpenQASM 3.0."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the results. For instance, the paper does not consider whether the QHE tasks disproportionately favor Qiskit-specific patterns seen in IBM's fine-tuning data, or whether model size differences confound comparisons."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@1 (unit test pass rate) and frames it as evaluating 'the correctness of the generated solutions' and 'functional correctness.' These are well-matched — no proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Models are identified by specific HuggingFace model IDs (e.g., 'ibm-granite/granite-8b-code-base'). Section IV names CodeLlama-34B-Python-hf, DeepSeek-Coder-33B-base, StarCoder2-15B, CodeGemma-7B with specific identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides 5 complete task examples (Examples 1-5) including full prompt text with function signatures, docstrings, canonical solutions, and test functions. The dataset structure format is described in Section III.A, and these prompts are what is fed to the models in HumanEval-style evaluation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1 is computed using greedy decoding (implying temperature=0). Fine-tuning hyperparameters are detailed: learning rate 1×10⁻⁵, cosine schedule, batch size 64, 3 epochs, 4096 context. Instruction tuning: learning rate 8×10⁻⁶, batch size 32, 3 epochs, 2048 sequence length."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is standard code completion (model generates function body from prompt)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV describes the fine-tuning data pipeline: Qiskit data collected April 2024, license-filtered (Apache 2.0, MIT, etc.), date-filtered (post-2023), deduplicated, cleaned to ~50M tokens. Instruction tuning data sources and sizes are also specified (8k openassistant, 5k commitpackft, 2.7k synthetic Q&A, 1k synthetic prompt/code)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section V (Discussion) focuses on future plans (OpenQASM 3.0 expansion, community contributions, version updates) rather than acknowledging methodological limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as the small number of difficult tasks, the lack of statistical testing, or the potential for benchmark bias toward IBM's training data distribution."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not discuss, for example, that pass@1 on 101 tasks does not measure real-world quantum programming productivity, or that results on Qiskit do not generalize to other quantum SDKs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The QHE dataset and model outputs were not released at time of publication. Only 5 example tasks are shown. Independent verification of the 101 tasks and model outputs is not possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes the dataset creation in detail: panel of quantum computing and Qiskit experts created tasks, each was peer-reviewed by every panel member, revised iteratively until consensus on 'correctness in terms of definition, feasibility, clarity, and reproducibility.'"
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The expert panel composition is described: 'Qiskit advocates, members of the Qiskit community, members of IBM Quantum support and documentation, and quantum computing software developers.' All are IBM-affiliated, which is disclosed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The task creation pipeline is documented: experts defined tasks → peer review by all panel members → revision based on feedback → consensus reached. The fine-tuning data pipeline is also documented with filtering stages and token counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is explicitly disclosed. The Acknowledgment section thanks specific individuals but does not mention grants, funding agencies, or corporate sponsorship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are clearly listed with IBM affiliations: IBM Research (Yorktown Heights, Zurich) and IBM Quantum (Yorktown Heights, Gurugram)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are IBM employees evaluating IBM's Granite model on IBM's Qiskit framework. IBM has a direct commercial interest in demonstrating that their model excels at Qiskit code generation. The funder (IBM) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The Granite model's Qiskit fine-tuning data was 'collected in April 2024' and the base model was 'pretrained on 4 Trillion tokens,' but no training cutoff date is stated for the base model or any of the baseline models (CodeLlama, DeepSeek-Coder, StarCoder2, CodeGemma)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section III states: 'The dataset is not part of the training dataset for the large language models (LLMs) being tested' and 'All tasks included in the dataset are original and not extracted from any existing tutorials, courses, books, or papers.' Section III.D also notes tasks involve 'novel features for which there is even more limited documentation.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper argues contamination is unlikely because tasks are hand-curated originals: 'While the experts responsible for creating the tasks may have been inspired by existing materials, they were required to write each task from scratch.' Some tasks test novel Qiskit 1.0 features with limited documentation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLMs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported for any of the evaluated models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Fine-tuning hyperparameters (batch size, epochs, learning rate) are stated but total GPU hours, hardware used, or wall-clock training time are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results use greedy decoding only (deterministic). No exploration of seed sensitivity for training runs or alternative decoding strategies (temperature, nucleus sampling)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table II caption states 'Pass@1 computed using greedy decoding,' which implies a single deterministic evaluation pass per model. This is sufficient for the reader to know the number of runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Fine-tuning hyperparameters are stated (learning rate, batch size, epochs) but no search budget is reported — it is unclear how these values were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents a single configuration for fine-tuning without explaining how it was selected or whether alternatives were considered."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Six models are compared across two benchmarks with no statistical tests applied, let alone multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "IBM employees evaluate IBM's Granite model against external baselines on IBM's benchmark. The inherent bias of evaluating one's own system on one's own benchmark is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The IBM model (8B parameters) is compared against models up to 34B parameters. No discussion of compute-performance tradeoffs or whether the domain-specific fine-tuning data gives an unfair advantage unrelated to model architecture."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether pass@1 on 101 hand-curated tasks actually measures the claimed capability of 'quantum code generation.' No analysis of whether the task distribution reflects real-world quantum programming needs or whether the 54/45/2 difficulty split is adequate."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — this is standard code completion evaluation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training cutoff dates are not stated for the baseline models. While the paper argues QHE tasks are novel, it does not explicitly analyze whether similar Qiskit code patterns in training data could enable models to solve tasks without genuine understanding."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (function signatures, docstrings, import statements) provides hints beyond what a practitioner would have in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether the 101 QHE tasks are independent or whether structural similarities between tasks (e.g., multiple circuit generation tasks sharing patterns) could inflate apparent performance."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is employed (no canary strings, membership inference, n-gram overlap analysis). The contamination argument relies solely on the assertion that tasks are original."
       }
     }
   },
   "claims": [
     {
       "claim": "Granite-8B-Code-QK achieves 46.53% pass@1 on QHE, outperforming all baselines including models up to 34B parameters.",
       "evidence": "Table II shows QHE pass@1 scores: Granite-8B-Code-QK (46.53%) vs DeepSeek-Coder-33B (39.6%), StarCoder2-15B (37.62%), Granite-8B-base (28.71%), CodeLlama-34B (26.73%), CodeGemma-7B (24.75%).",
       "supported": "moderate"
     },
     {
       "claim": "Domain-specific fine-tuning on Qiskit data improves QHE performance by 17.8 percentage points.",
       "evidence": "Table II: Granite-8B-Code-Base (28.71%) vs Granite-8B-Code-QK (46.53%). Section IV states 'The pass rate jumps 17.8 points.'",
       "supported": "moderate"
     },
     {
       "claim": "The QHE dataset is not contaminated by training data of the tested models.",
       "evidence": "Section III states tasks are 'original and not extracted from any existing tutorials, courses, books, or papers' and 'not part of the training dataset for the large language models being tested.' No verification method is used.",
       "supported": "weak"
     },
     {
       "claim": "No current model can solve the difficult-level tasks.",
       "evidence": "Table III shows 0/2 difficult tasks passed by all 6 models.",
       "supported": "strong"
     },
     {
       "claim": "Qiskit-specific fine-tuning only slightly degrades general HumanEval performance.",
       "evidence": "Table II: Granite-8B-Code-Base HE score 39.02% vs Granite-8B-Code-QK 38.41%, a drop of 0.61 percentage points.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own product on own benchmark",
       "detail": "All authors are IBM employees. They created the QHE benchmark, fine-tuned the Granite model on IBM's Qiskit data, and evaluated it against external competitors on their own benchmark. IBM has a direct commercial interest in demonstrating Granite excels at Qiskit code generation."
     },
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "Model comparisons rely on single-run greedy decoding pass@1 with no error bars, confidence intervals, or significance tests. The claim that Granite-8B-QK 'performs better than all the other baselines' is based on raw number comparison only."
     },
     {
       "flag": "Tiny difficult category renders it meaningless",
       "detail": "Only 2 of 101 tasks are labeled 'difficult.' With all models scoring 0/2, this category has no discriminative power and the difficulty distribution is heavily skewed toward basic (54) and intermediate (45)."
     },
     {
       "flag": "Unfair model size comparison",
       "detail": "Granite-8B-QK (8B, domain-fine-tuned) is compared against generic models up to 34B parameters without domain-specific fine-tuning. The paper does not discuss whether the comparison is fair given the asymmetric fine-tuning."
     },
     {
       "flag": "Dataset and code not released at publication",
       "detail": "The benchmark is promised for 'Q3 2024' release but was unavailable at time of publication, preventing independent verification of all claims."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen"],
       "year": 2021,
       "relevance": "Introduced HumanEval, the code generation benchmark that QHE is modeled after."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu"],
       "year": 2023,
       "relevance": "EvalPlus — improved version of HumanEval/MBPP with more rigorous test cases for code generation evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin"],
       "year": 2021,
       "relevance": "Introduced MBPP benchmark for program synthesis, a foundational code generation evaluation."
     },
     {
       "title": "Ds-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Y. Lai"],
       "year": 2022,
       "relevance": "Domain-specific code generation benchmark using Python data science SDKs, architecturally similar to QHE's approach."
     },
     {
       "title": "Multipl-e: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["F. Cassano"],
       "year": 2022,
       "relevance": "Multilingual code generation benchmark enabling cross-language evaluation of LLMs."
     },
     {
       "title": "Qiskit Code Assistant: Training LLMs for generating Quantum Computing Code",
       "authors": ["N. Dupuis"],
       "year": 2024,
       "arxiv_id": "2405.19495",
       "relevance": "Companion paper on training LLMs specifically for Qiskit code generation."
     },
     {
       "title": "OctoPack: Instruction Tuning Code Large Language Models",
       "authors": ["N. Muennighoff"],
       "year": 2023,
       "relevance": "Instruction tuning methodology for code LLMs, directly used for QK model's instruct-tuning stage."
     },
     {
       "title": "Granite code models: A family of open foundation models for code intelligence",
       "authors": ["M. Mishra"],
       "year": 2024,
       "arxiv_id": "2405.04324",
       "relevance": "Describes the base Granite-8B model family used as the foundation for the QK model."
     },
     {
       "title": "CruxEval: A benchmark for code reasoning, understanding and execution",
       "authors": ["A. Gu"],
       "year": 2024,
       "relevance": "Code reasoning benchmark testing LLM understanding and execution prediction capabilities."
     },
     {
       "title": "Can it edit? Evaluating the ability of large language models to follow code editing instructions",
       "authors": ["F. Cassano"],
       "year": 2023,
       "relevance": "Benchmark evaluating LLMs on code editing tasks, complementary to code generation evaluation."
     },
     {
       "title": "CrossCodeEval: A diverse and multilingual benchmark for cross-file code completion",
       "authors": ["Y. Ding"],
       "year": 2023,
       "relevance": "Cross-file code completion benchmark testing LLMs on multi-file context understanding."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Quantum computing programming is a niche domain; the benchmark is useful only to researchers working at the intersection of LLMs and quantum computing."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Domain-specific fine-tuning improving domain-specific benchmark performance is entirely expected."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper is about quantum code generation quality."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward benchmark introduction and evaluation."
     },
     "demo_ability": {
       "score": 1,
       "justification": "The Granite base model is available on HuggingFace (ibm-granite/granite-8b-code-base) but the QHE dataset and fine-tuned QK model were not yet released."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "IBM Quantum is a well-known brand in quantum computing, and Qiskit is a leading quantum SDK, though neither has mainstream AI/ML visibility."
     }
   }
 }