ai-research-survey

scan.json (26734B)

---

 {
   "paper": {
     "title": "Enhancing LLM-based Quantum Code Generation with Multi-Agent Optimization and Quantum Error Correction",
     "authors": [
       "Charlie Campbell",
       "Hao (Mark) Chen",
       "Wayne Luk",
       "Hongxiang Fan"
     ],
     "year": 2025,
     "venue": "Design Automation Conference",
     "arxiv_id": "2504.14557",
     "doi": "10.1109/DAC63849.2025.11133316"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "A multi-agent framework for quantum code generation using StarCoder with CoT, SCoT, RAG, and QEC agents. Structured CoT prompting yields the largest improvement (~40 percentage points on a custom test suite), while RAG provides only marginal benefit (~4%). Multi-pass inference helps fix syntactic errors but has diminishing returns beyond 3 passes. QEC integration is demonstrated only in simulation, not on actual quantum hardware.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or supplementary materials link is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The training dataset (scraped GitHub Qiskit repos) and custom test suite are not released. No download links provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using the transformers library and LoRA adapter but provides no requirements.txt, Dockerfile, or library versions. Not enough detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts are provided. The paper describes the methodology but not step-by-step reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Figure 3 and Table I are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'CoT leading to an increase of 32%' and 'SCoT an increase of 40%' are made without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context: fine-tuned model achieves 28%, CoT adds 32pp, SCoT adds 40pp. Table I shows absolute accuracy figures for each configuration (17.9% to 46.5%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test suite composition is described (47% basic, 24% intermediate, 29% advanced) but the total number of test prompts is never stated, and no justification for the sample size is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or results across multiple runs are reported. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The fine-tuned StarCoder model without optimizations serves as a baseline. Table I also compares against the IBM Granite-20B-CODE Qiskit Assistant model."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "IBM Qiskit Code Assistant (2024) and StarCoder2 (2024) are both contemporary baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Each optimization technique (RAG, CoT, SCoT, multi-pass) is evaluated independently in Figure 3, isolating the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported using both syntactic accuracy and semantic accuracy (Section V-C: '45.7% syntactic accuracy, but only 33.8% semantic accuracy'), as well as pass@k on the Qiskit HumanEval benchmark."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via test suites and pass@k metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The custom test suite and Qiskit HumanEval benchmark are used for evaluation, but there is no explicit discussion of whether these were kept separate from any development/tuning decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 breaks down results by technique, and Section V-C separates syntactic vs semantic accuracy. The test suite is categorized into basic (47%), intermediate (24%), and advanced (29%) tiers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C and V-D discuss failure modes: RAG's failure attributed to outdated documentation, multi-pass diminishing returns due to import/deprecation errors, and incorrect CoT prompt generation causing semantically invalid code."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RAG yielding only 4% improvement is prominently reported as a negative finding. Multi-pass inference showing diminishing returns beyond triple passes is also reported as a limitation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'structured CoT significantly improve the generation of quantum algorithms by up to 50%' but the results section reports SCoT improvement of 40 percentage points (Section V-C). The 50% figure does not appear in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'CoT improves accuracy' are supported by controlled single-variable ablation: each technique is evaluated independently on the same base model and test suite (Figure 3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'LLM-based Quantum Code Generation' broadly, but experiments are limited to StarCoder-3B/7B on Qiskit only. The conclusion states 'accurate, fault-tolerant quantum computing code generation using Large Language Models' without bounding to the tested models or framework."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E discusses why RAG failed (outdated Qiskit documentation, basic splitting technique), why CoT outperformed RAG (more direct model guidance vs. inference from limited data), and why multi-pass had diminishing returns (root cause is import/deprecation errors)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures syntactic and semantic accuracy on a test suite and frames results in terms of code generation accuracy. Claims match the granularity of measurements without overreaching into broader framing like 'developer productivity.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Primary models are specified: StarCoder-3B, Starcoder2-7B, IBM Granite-20B-CODE (Section V-A, Table I). These are specific open-source model checkpoints with deterministic weights. GPT-4o used for CoT prompt generation lacks a snapshot date but is not the evaluated model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described in natural language ('we manually created the first 5 prompts from our testing set using the same techniques demonstrated in previous code generation work') but the actual prompt text is not provided. The multi-pass prompt template is described but not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Training hyperparameters are reported (1500 steps, batch size 4, learning rate 3×10⁻⁴, 100 warmup steps, cosine scheduler, FIM rate 0.1, LoRA). However, inference parameters (temperature, top-p, max tokens) for code generation are not stated despite using pass@k which requires sampling."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent framework is described in Section III with Figure 1: three agents (code generator, semantic analyzer, QEC decoder), their roles, and interaction flow including multi-pass refinement with error traces."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B describes the pipeline: GitHub scraping with open-source filter → date filtering (post-Feb 2024) → Qiskit import filtering → notebook splitting by sentinel tokens → upsampling from 3M to 9M tokens with priority weighting."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E 'Observation and Future Directions' discusses multiple limitations: limited dataset sizes, RAG documentation issues, topology-specific QEC decoder. While not titled 'Limitations', it contains substantive limitation discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: limited dataset evidenced by 'poor increase in the fine-tuned models' pass@1 accuracy', RAG technique limited by 'basic RAG splitting technique which does not take into account code structure', topology-specific QEC decoder 'requires retraining each time you want to run the code on a different device.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Limitations are framed as 'future work' rather than explicit scope boundaries. No statements like 'our results are limited to Qiskit/StarCoder and should not be generalized to other quantum frameworks or models.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (training set, test prompts, model outputs) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B describes data collection: scraping GitHub repositories with open-source licenses, filtering by update date (after Feb 2024), filtering for Qiskit imports, splitting notebooks by sentinel tokens."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from GitHub repositories, not a standard benchmark, but the scraping methodology is described in Section III-B."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the filtering criteria are described, counts at each stage are missing. The paper jumps from 'scraping Github repositories' to '3M tokens' without stating how many repositories were found, how many survived each filter, or how many files were retained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section lists UK EPSRC grants (EP/W032635/1, EP/V028251/1, EP/S030069/1, EP/X036006/1) and support from Intel and AMD."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are affiliated with the Department of Computing, Imperial College London. They are not evaluating their own commercial product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "EPSRC is an independent research council. Intel and AMD provide support but the paper evaluates open-source models (StarCoder), not Intel/AMD products. Funders have no direct financial stake in the results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The fine-tuning data is filtered to after Feb 2024, but the StarCoder base model's pre-training data cutoff is not stated. It is unclear what the base model may have seen."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Qiskit HumanEval benchmark problems or their custom test prompts could have appeared in StarCoder's pre-training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Qiskit HumanEval was published in 2024. StarCoder2 was also released in 2024. No discussion of whether benchmark problems were in the training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token consumption is reported for the multi-agent framework despite using multi-pass inference and multiple agents."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training is described (1500 steps, batch size 4) but no GPU hours, hardware specs, wall-clock time, or total compute budget is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using pass@k and creating 'a sample of results' but does not state the exact number of runs or samples generated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We found that the optimal FIM rate was 0.1' without reporting how many configurations were tried or the search method used."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The FIM rate of 0.1 is stated as 'optimal' without explaining the selection process or validation set used. Model choice (StarCoder-3B) is justified only as 'suitable for adapting to new programming languages.'"
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so correction for multiple comparisons is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own framework against IBM's model without acknowledging author-evaluation bias. Their test suite was created by them for their system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Multi-pass inference requires multiple forward passes but the compute cost is not quantified. No performance-vs-compute comparison is provided despite the multi-agent framework clearly requiring more compute than a single-pass baseline."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper creates a custom test suite and uses Qiskit HumanEval without discussing whether these benchmarks adequately measure quantum code generation capability. No construct validity analysis."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Table I compares StarCoder2-7B variants (with RAG/CoT) against IBM Granite-20B-CODE without addressing that differences could stem from model size (7B vs 20B), training data, or scaffolding rather than the techniques themselves."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the StarCoder base model's pre-training data could have included Qiskit HumanEval problems or algorithms tested in the custom suite."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information, e.g., whether CoT prompts generated by GPT-4o could contain implicit solutions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between scraped GitHub training data and test suite prompts, which may test algorithms present in the training repositories."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method (canary strings, membership inference, decontamination) is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Structured CoT significantly improves quantum code generation by up to 50%",
       "evidence": "Figure 3 and Section V-C show SCoT improvement of 40 percentage points over the fine-tuned model on the custom test suite. The 50% figure cited in the abstract does not appear in the results section.",
       "supported": "weak"
     },
     {
       "claim": "RAG shows limited improvement, yielding an accuracy increase of only 4%",
       "evidence": "Figure 3 and Section V-C confirm RAG's marginal impact. Attributed to outdated Qiskit documentation and basic RAG splitting technique.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-pass inference improves accuracy to 34% using triple passes, with diminishing returns beyond that",
       "evidence": "Section V-D states this result. Root cause identified as import/deprecation errors that persist across passes.",
       "supported": "moderate"
     },
     {
       "claim": "The QEC agent reduces quantum noise in generated programs",
       "evidence": "Figure 4 shows the Deutsch-Jozsa oracle example, but (c) is explicitly simulated: 'we simulated our results for (c) using a lower error probability than IBM Brisbane, corresponding to the new error rate after QEC.' Not demonstrated on actual hardware.",
       "supported": "weak"
     },
     {
       "claim": "Fine-tuning on Qiskit data increases pass@1 by 10%, up to 28% overall",
       "evidence": "Section V-B reports this improvement on the custom test suite. Table I shows 17.9% → 24.5% on Qiskit HumanEval for StarCoder2-7B.",
       "supported": "moderate"
     },
     {
       "claim": "CoT prompting approaches the performance of the much larger IBM Qiskit model (20B)",
       "evidence": "Table I shows StarCoder2-7B-QKCoT at 41.4% vs IBM Granite-20B-CODE-QK at 46.5% on Qiskit HumanEval, a ~5% gap.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance on any result",
       "detail": "All results in Figure 3 and Table I are single point estimates with no confidence intervals, standard deviations, or variance across runs, making it impossible to assess result stability."
     },
     {
       "flag": "QEC results are simulated, not demonstrated on real hardware",
       "detail": "The paper states 'we simulated our results for (c) using a lower error probability than IBM Brisbane, corresponding to the new error rate after QEC.' The core QEC claim is based on simulation, not actual quantum error correction applied on hardware."
     },
     {
       "flag": "Abstract overstates improvement",
       "detail": "The abstract claims 'up to 50%' improvement from structured CoT, but the results section shows 40 percentage points. The 50% figure is not traceable to any specific result in the paper."
     },
     {
       "flag": "Test suite size unstated",
       "detail": "The total number of test prompts is never stated — only proportions (47% basic, 24% intermediate, 29% advanced). A small test suite could produce unreliable accuracy estimates."
     },
     {
       "flag": "Unfair cross-model comparison",
       "detail": "Table I compares a 7B parameter model (StarCoder2-7B with optimizations) against a 20B parameter model (IBM Granite), yet frames the 5% gap as 'approaching' parity without acknowledging the 3x size difference or controlling for compute."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentCoder: Multiagent-code generation with iterative testing and optimisation",
       "authors": ["D. Huang", "Q. Bu", "J. M. Zhang", "M. Luck", "H. Cui"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent framework for code generation, direct predecessor and inspiration for this paper's approach."
     },
     {
       "title": "Starcoder 2 and the stack v2: The next generation",
       "authors": ["A. Lozhkov", "R. Li", "L. Ben Allal"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Base code generation model used for fine-tuning in the paper's experiments."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["B. Rozière", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code generation model family discussed as related work."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and pass@k metric used in this paper's evaluation."
     },
     {
       "title": "Qiskit Code Assistant: Training LLMs for generating quantum computing code",
       "authors": ["N. Dupuis", "L. Buratti"],
       "year": 2024,
       "arxiv_id": "2405.19495",
       "relevance": "IBM's quantum code generation model serving as the primary baseline in Table I."
     },
     {
       "title": "A survey on large language models for code generation",
       "authors": ["J. Jiang", "F. Wang", "J. Shen", "S. Kim", "S. Kim"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Survey of LLM code generation covering the broader landscape this paper contributes to."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans"],
       "year": 2022,
       "relevance": "Foundational prompting technique that produced the largest accuracy improvement in this paper's experiments."
     },
     {
       "title": "Structured chain-of-thought prompting for code generation",
       "authors": ["J. Li", "G. Li", "Y. Li", "Z. Jin"],
       "year": 2023,
       "relevance": "SCoT technique applied in the paper showing 40pp improvement in quantum code generation accuracy."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning method used to adapt StarCoder for quantum code generation."
     },
     {
       "title": "Qiskit HumanEval: An evaluation benchmark for quantum code generative models",
       "authors": ["S. Vishwakarma", "F. Harkins", "S. Golecha"],
       "year": 2024,
       "relevance": "Quantum-specific code generation benchmark used for evaluation in Table I."
     },
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["C. Snell", "J. Lee", "K. Xu", "A. Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Inference-time compute scaling research that motivates the paper's multi-pass optimization approach."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "Major LLM referenced as context for model capabilities; GPT-4o used for CoT prompt generation."
     }
   ]
 }