ai-research-survey

scan.json (29904B)

---

 {
   "paper": {
     "title": "PennyLang: Pioneering LLM-Based Quantum Code Generation with a Novel PennyLane-Centric Dataset",
     "authors": [
       "Abdul Basit",
       "Nouhaila Innan",
       "Muhammad Haider Asif",
       "Minghao Shao",
       "Muhammad Kashif",
       "Alberto Marchisio",
       "Muhammad Shafique"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.02497",
     "doi": "10.48550/arXiv.2503.02497"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PennyLang is a 3,347-sample PennyLane-centric dataset curated from GitHub, textbooks, and documentation. When paired with RAG, it substantially boosts open-source model performance (Qwen 7B from 8.7% to 41.7% success rate), while commercial models like Claude 3.5 Sonnet (95.1% baseline) show no benefit from retrieval augmentation. The 75% context setting consistently outperformed full-context retrieval, suggesting moderate retrieval reduces noise. No statistical tests, error bars, or contamination analysis were performed.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We release both the PennyLang dataset and the evaluation framework' in the conclusion, but no repository URL, Zenodo archive, or HuggingFace link is provided anywhere in the paper text."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the dataset is 'open-source' and 'off-the-shelf' but provides no download URL or repository link for the PennyLang dataset itself. Promises of release without a working URL count as NO."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment specification is provided. The paper mentions specific tools (LangChain, Chroma, OpenAI Embeddings) but not their versions or installation requirements."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology describes the pipeline conceptually but does not include commands, scripts, or a README for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table VI reports only point estimates for success rates, Pass@1 through Pass@5, and working notebook counts. No confidence intervals, error bars, or uncertainty measures are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'RAG substantially improves performance' and makes comparative claims across models and retrieval settings, but no statistical significance tests (t-tests, bootstrap, etc.) are performed. Differences are assessed by comparing raw numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context: 'Qwen 7B's success rate rises from 8.71% to 41.66% (≈32.95% relative increase)' and 'LLaMa 4 improves from 78.78% to 84.84% (≈7.7% relative increase).' The from-to framing provides sufficient context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 264 test cases and generates 1,320 notebooks per model (264 × 5). No justification is given for why 264 test cases were chosen or whether this is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Pass@5 involves 5 generations per test case as part of the metric, but no indication of running the full experiment multiple times to measure stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares RAG-augmented vs. non-augmented settings (0%, 50%, 75%, 100% context) and evaluates across 4 models (Qwen 7B, LLaMa 4, GPT-4o-mini, Claude 3.5 Sonnet), providing baseline comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models are contemporary: GPT-4o Mini, Claude 3.5 Sonnet, Qwen 2.5-7B-Instruct-Turbo, and LLaMa 4 Maverick 17B. These represent recent open-source and commercial models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates the retrieval context coverage (0%, 50%, 75%, 100%), showing how different amounts of retrieved context affect performance. This reveals that 75% often outperforms 100%."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table VI reports both 'Working Notebooks' (syntax/execution correctness — does the notebook run?) and 'Success Rate' / Pass@k (functional correctness — does it pass tests?). These capture different aspects of code quality."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via test execution (Pass@k). Human verification was used during dataset construction but not for evaluating LLM outputs. No human ratings of output quality, readability, or correctness beyond automated tests."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated benchmark suite of 264 test cases was designed to evaluate PennyLane functionalities. These are separate from the 3,347 dataset samples used for RAG retrieval."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Table VI breaks down results by model and context level, but provides no per-category breakdown by quantum functionality type (e.g., gates vs. optimization vs. chemistry). Section IV analyzes dataset statistics by feature group, but evaluation results are only aggregate."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No error analysis or qualitative examination of failure cases is provided. The paper notes that commercial models degrade with full retrieval but does not analyze specific failing test cases or common error patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that GPT-4o-mini and Claude 3.5 Sonnet performance degrades with full-context RAG (e.g., Claude drops from 95.07% without RAG to 89.01% with full retrieval). The paper also notes full context can introduce 'redundancy or irrelevant tokens.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported by Table VI: Qwen 7B rises from 8.71% to 41.66% (abstract says 8.7% to 41.7%), LLaMa 4 from 78.78% to 84.84% (abstract says 78.8% to 84.8%). The claim about commercial models not benefiting is also supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claim is that RAG with PennyLang improves code generation. The study design (same models tested with and without RAG using the same test cases) is a controlled comparison adequate for this claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Pioneering LLM-Based Quantum Code Generation' broadly, but results are limited to PennyLane code with 4 specific models on 264 test cases. No other quantum frameworks (Qiskit, Cirq) are tested. The abstract claims to advance 'AI-assisted quantum development' generally from PennyLane-only evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses why commercial models don't benefit ('broad pretraining, which likely includes similar material') and why partial context outperforms full context ('focused and moderately sized context windows improve model grounding by avoiding redundancy and context saturation'). These address alternative explanations for observed patterns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures functional correctness via test execution (Pass@k) and directly frames results as code correctness. The measurement matches the claim granularity without inflating to broader constructs like 'developer productivity.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o Mini' and 'Claude 3.5 Sonnet' without snapshot dates or API versions. While 'Qwen2.5-7B-Instruct-Turbo' and 'LLaMa 4 Maverick 17B' are more specific, the commercial model versions are insufficient per the schema (marketing names without snapshot dates)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using ChatPromptTemplate for 'structured prompts' (Table V) and describes the RAG pipeline conceptually, but never provides the actual prompt text sent to the models. Only descriptions like 'structured prompts for AI models to ensure consistency' are given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other inference hyperparameters are reported for any of the four evaluated models. These significantly affect output quality and reproducibility."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The RAG pipeline is described in detail: LangChain for orchestration, Chroma vector database for storage, OpenAI Embeddings for encoding, MMR-based retrieval, and ChatPromptTemplate for formatting (Section III.C, Table V, Fig. 2). The full pipeline from query to response is diagrammed."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A documents the full preprocessing pipeline: GitHub API search with filtering criteria (main branch only, no forks, Python files only, import statements checked), duplicate removal, PEP 8 formatting, manual review, and GPT-4o conversion to instruction-query format."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The conclusion mentions future directions (cross-framework generalization, lightweight fine-tuned models) but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues such as test case representativeness, dataset bias toward certain PennyLane patterns, or generalizability concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not articulate what the results do NOT show, such as limitations to specific quantum computing paradigms, model sizes, or programming tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 3,347 code samples and 264 test cases are claimed to be released but no download URL is provided. The generated notebooks (1,320 per model) are not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A describes data collection in detail: GitHub API search for repos with 'PennyLane' in name/description/README, filtering by license type, extraction from two specific quantum computing textbooks, and scraping official PennyLane documentation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Data source selection criteria are well-described: GitHub repos required permissive open-source licenses, PennyLane keyword presence, main branch at latest commit, no forks, Python files only with explicit import statements. Books and documentation sources are named specifically."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw sources to final dataset is documented: collection (GitHub API, books, docs) → filtering (import checks, license verification) → preprocessing (license/metadata removal, PEP 8 formatting) → annotation (GPT-4o conversion) → deduplication. Table III shows sample counts by source (1,952 + 1,321 + 53 + 21 = 3,347)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section discloses funding from NYUAD Center for Quantum and Topological Systems (CQTS) via Tamkeen grant CG008, and Center for CyberSecurity (CCS) via Tamkeen grant G1104."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: NYU Abu Dhabi eBRAIN Lab and CQTS. The authors are not affiliated with PennyLane/Xanadu or the LLM providers being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (Tamkeen/NYUAD research institutes) are academic funding bodies with no financial stake in PennyLane adoption, LLM performance, or quantum computing framework preferences."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four evaluated models (GPT-4o Mini, Claude 3.5 Sonnet, Qwen 7B, LLaMa 4). This is critical since PennyLane documentation is public and likely in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 264 test cases or PennyLane code patterns could have appeared in the models' training data. The dataset is curated from public sources (GitHub, official docs) highly likely to be in model training sets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No contamination analysis is performed despite the dataset being drawn from publicly available PennyLane documentation and GitHub repositories that these models were almost certainly trained on. This is a critical omission."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a dataset curation and benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study involves code collection and automated model evaluation only."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation generates 1,320 notebooks per model (5,280 total) using both commercial APIs (GPT-4o Mini, Claude 3.5) and open-source models. No API costs, token counts, or inference latency is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgment mentions NYU Abu Dhabi HPC resources, but no GPU hours, total API spend, or compute time is quantified for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No random seeds or seed sensitivity analysis is mentioned. LLM generation is stochastic (temperature-dependent), and results could vary across runs, but this is not examined."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'each model generates five candidate solutions per test' for Pass@5, and '1320 notebooks were generated and executed per model (264 tests × 5 completions).' The number of generations is clearly documented."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described for the RAG pipeline (e.g., chunk size, embedding model, MMR parameters) or for inference settings. The retrieval context percentages (0%, 50%, 75%, 100%) appear chosen without justification."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All four context configurations (0%, 50%, 75%, 100%) are reported for all four models in Table VI. The paper does not selectively report only the best configuration — all results are visible."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all (no p-values, no hypothesis testing), so the question of correcting for multiple comparisons does not arise."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed both the dataset and the 264-test benchmark, then evaluated their own RAG pipeline against baselines. No acknowledgment of author-evaluation bias or independent evaluation is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different retrieval settings (0% to 100% context) imply different token counts and compute costs, but performance is not analyzed as a function of compute budget. No token-level cost comparison across conditions."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the 264 test cases adequately measure quantum code generation capability vs. PennyLane API memorization. No analysis of what the benchmark actually tests relative to the claimed evaluation goal."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All four models are evaluated within the same RAG scaffold (LangChain + Chroma + MMR), ensuring that model comparisons are not confounded by different scaffolding. The same retrieval pipeline is used consistently."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The PennyLane documentation and GitHub repositories used to build the dataset have been public for years and are almost certainly in the training data of the evaluated commercial models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the RAG-retrieved context provides information that would not be available in realistic deployment settings, or whether retrieved examples contain patterns too similar to the test cases."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 264 test cases and the 3,347 dataset samples share structural similarities, come from the same sources, or overlap in content. Both are drawn from the PennyLane ecosystem, creating potential non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No n-gram overlap analysis, deduplication check between dataset and test cases, or contamination testing for the evaluated models."
       }
     }
   },
   "claims": [
     {
       "claim": "RAG with PennyLang substantially improves Qwen 7B performance from 8.71% to 41.66% success rate on PennyLane code generation tasks.",
       "evidence": "Table VI shows Qwen 7B at 0% context achieves 8.71% success rate (23 pass@5) vs. 41.66% (110 pass@5) at full context. Section V reports these numbers.",
       "supported": "moderate"
     },
     {
       "claim": "LLaMa 4 Maverick improves from 78.78% to 84.84% success rate with full-context RAG augmentation.",
       "evidence": "Table VI shows LLaMa 4 at 0% context = 78.78% (208 pass@5) and full context = 84.84% (224 pass@5). The improvement is modest at ~6 percentage points.",
       "supported": "moderate"
     },
     {
       "claim": "Commercial models (GPT-4o-mini, Claude 3.5 Sonnet) do not benefit from RAG augmentation with PennyLang; their best performance is achieved without retrieval.",
       "evidence": "Table VI shows GPT-4o-mini peaks at 85.60% without RAG vs. 80.30% with full RAG, and Claude 3.5 Sonnet peaks at 95.07% without RAG vs. 89.01% with full RAG. Both degrade with full-context retrieval.",
       "supported": "strong"
     },
     {
       "claim": "75% context retrieval often outperforms full-context (100%) retrieval, suggesting moderate retrieval reduces noise.",
       "evidence": "Table VI: Qwen 7B 75% = 45.07% vs. full = 41.66%; GPT-4o-mini 75% = 84.84% vs. full = 80.30%; Claude 3.5 75% = 92.44% vs. full = 89.01%. Pattern is consistent across 3 of 4 models.",
       "supported": "moderate"
     },
     {
       "claim": "PennyLang is the first PennyLane-centric dataset for LLM-based quantum code generation.",
       "evidence": "Table I compares existing datasets, showing none are structured, curated PennyLane datasets. Existing PennyLane resources (documentation, GitHub repos) are unstructured. Table II positions PennyLang as a novel contribution.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical rigor",
       "detail": "All comparative claims are made by comparing raw numbers without statistical tests, confidence intervals, or variance measurements. With stochastic LLM generation and only 264 test cases, observed differences could be within noise."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "PennyLane documentation and GitHub repositories (the dataset sources) are almost certainly in the training data of GPT-4o-mini and Claude 3.5 Sonnet. The high baseline performance of commercial models without RAG (85.6% and 95.1%) may reflect memorization, not capability. No contamination analysis is performed."
     },
     {
       "flag": "Circularity in dataset construction",
       "detail": "GPT-4o was used to convert code snippets into instruction-query pairs, then GPT-4o-mini (a variant from the same model family) is evaluated on the resulting dataset. This creates potential circularity where the evaluation model benefits from the formatting model's patterns."
     },
     {
       "flag": "No artifacts despite open-source claims",
       "detail": "The paper repeatedly claims the dataset and evaluation framework are 'open-source' and 'released,' but provides no URL, repository link, or download location anywhere in the paper."
     },
     {
       "flag": "Author-designed benchmark without independent validation",
       "detail": "The 264 test cases were designed by the same team that built the dataset. No independent validation of the benchmark's coverage, difficulty distribution, or construct validity is provided."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries. This omission is notable for a paper making broad claims about 'pioneering' quantum code generation."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for LLM code generation evaluation, foundational to code generation assessment methodology."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Y. Li", "D. R. Zhou", "N. Kohl"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Demonstrates LLM performance on competitive programming, relevant to understanding LLM code generation capabilities and limits."
     },
     {
       "title": "OpenAI Codex: Programming with Natural Language",
       "authors": ["OpenAI"],
       "year": 2021,
       "relevance": "Pioneering work on LLM-driven code generation from natural language, establishing the paradigm this paper extends to quantum computing."
     },
     {
       "title": "Code LLaMA: Open Foundation Models for Code",
       "authors": ["B. Rozière", "J. Gehring", "F. Gloeckle"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code generation model relevant to understanding how specialized fine-tuning affects code generation quality."
     },
     {
       "title": "Qiskit Code Assistant: Training LLMs for Generating Quantum Computing Code",
       "authors": ["N. Dupuis", "L. Buratti", "S. Vishwakarma"],
       "year": 2024,
       "relevance": "Most directly related prior work: adapts LLMs for quantum code generation using Qiskit, the closest precedent to PennyLang's PennyLane approach."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained model for code understanding that established bidirectional transformer approaches for programming language processing."
     },
     {
       "title": "InCoder: A Generative Model for Code Infilling and Synthesis",
       "authors": ["D. Fried", "A. Aghajanyan", "J. Lin"],
       "year": 2023,
       "arxiv_id": "2204.05999",
       "relevance": "Multi-turn interactive code generation model relevant to understanding iterative code completion approaches."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus"],
       "year": 2021,
       "arxiv_id": "2005.11401",
       "relevance": "Foundational RAG paper that this work builds on to augment LLMs with domain-specific quantum computing knowledge."
     },
     {
       "title": "QCircuitNet: A Large-Scale Hierarchical Dataset for Quantum Algorithm Design",
       "authors": ["R. Yang", "Y. Gu", "Z. Wang"],
       "year": 2024,
       "arxiv_id": "2410.07961",
       "relevance": "Quantum computing dataset for LLM training (Qiskit-focused), directly comparable to PennyLang's contribution for PennyLane."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["T. B. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing few-shot learning capabilities of large language models, foundational to the LLM code generation paradigm."
     },
     {
       "title": "A Survey on Large Language Models for Code Generation",
       "authors": ["J. Jiang", "F. Wang", "J. Shen"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Recent survey of LLM code generation relevant to understanding the landscape of AI-assisted programming research."
     }
   ]
 }