ai-research-survey

scan-v5.json (27230B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Leveraging Mutation Analysis for LLM-based Repair of Quantum Programs",
     "authors": [
       "Chihiro Yoshida",
       "Yuta Ishimoto",
       "Olivier Nourry",
       "Masanari Kondo",
       "Makoto Matsushita",
       "Yasutaka Kamei",
       "Yoshiki Higo"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.12273",
     "doi": "10.48550/arXiv.2601.12273"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: low repair success in prior work (Guo et al. 17%, HornBro 249-gate cost) is cited; mutation analysis effectiveness shown in Table I (94.4% for S+D+M); explanation quality improvements shown in Table II.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a controlled ablation design (4 prompt configurations: S, S+D, S+M, S+D+M) isolating the effect of mutation analysis. This design is appropriate for causal inference within the LLM prompt engineering context.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Scope is explicitly bounded: 18 bugs from Bugs4Q, Qiskit only, simulator only, GPT-5 only. External validity section (VI) clearly lists limitations regarding broader applicability to different frameworks, LLMs, and real quantum hardware.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Paper discusses why S+M underperforms (mutation analysis requires tests, so D is available anyway) and why WO vs TE bugs respond differently (runtime vs error info). However, limited discussion of why mutation analysis mechanistically helps the LLM.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Repair success is measured as 'passes all tests', which directly tests the repair outcome. Explanation quality is measured against ground-truth patches (correctness, completeness, complexity), matching the claim about explanation improvement.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' provides dedicated discussion of construct, internal, and external validity with specific threats rather than boilerplate disclaimers.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: test-passing repairs may not match developer intent (construct); stochastic LLM outputs and manual evaluation subjectivity (internal); simulator vs hardware, single benchmark, single framework, single LLM (external).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit boundaries: 18 bugs (not 42), Qiskit only, simulator only, GPT-5 only, no human subjects, no real quantum hardware. All stated in introduction and threats section.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments disclose 5 funding sources: JSPS Grants-in-Aid (grants JP25K03102, JP24H00692, JP23K24823), JST ASPIRE (JPMJAP2415), and Inamori Research Institute fellowship.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list institutional affiliations: University of Osaka (4 authors) and Kyushu University (3 authors). No undisclosed affiliations with quantum computing vendors or OpenAI.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Japanese government and academic organizations (JSPS, JST, Inamori), independent of OpenAI/GPT-5 and quantum computing companies. Authors evaluate external tools they do not control.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No explicit competing interests statement or declaration of patents, equity, or consulting relationships. While likely none exist, absence of explicit statement is a gap.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms defined: APR (Automated Program Repair), mutation analysis ('evaluates how small changes affect execution'), LLM (GPT-5), quantum concepts (qubits, gates, superposition, entanglement).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions explicitly stated: (1) first evidence mutation analysis improves LLM-based quantum APR; (2) dynamic info + mutation analysis yields highest repair rate; (3) mutation analysis improves explanation quality.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section II systematically engages prior work: classical APR (InferFix, hierarchical knowledge injection), quantum APR (Guo et al. 17%, UnitAR, HornBro 249-gate cost), and explains why LLM-based approach chosen over synthesis.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Data availability statement (VIII) claims 'all data, benchmarks, scripts, and prompts' available in replication package [29] with Zenodo DOI 10.5281/zenodo.17626083.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "Bugs4Q benchmark is publicly available. QMutPy outputs are deterministic. Static information and prompts promised in replication package. Mutation analysis results documented in Fig 1.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Mentions Python, Qiskit, QMutPy, GPT-5 API but no requirements.txt, Dockerfile, or version pinning. Hyperparameters use 'default settings' (vague). Insufficient for reproducibility.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": true,
           "justification": "Process is documented: clone Bugs4Q, apply QMutPy, construct 4 prompt configs per Fig 1, call GPT-5 API 5 times per config, evaluate outputs. Referred to replication package for exact prompts/scripts.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Table I reports success rates (%) with no confidence intervals. Table II reports counts with no error bars or uncertainty quantification. Generated 5 outputs per config but reports only success binary outcome.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Differences between configurations (77.8% S vs 88.9% S+D vs 94.4% S+D+M) are not tested for statistical significance. No p-values or significance thresholds reported.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes implicit in percentages (e.g., 94.4% vs 77.8% = 16.6pp improvement). Not formally reported as Cohen's d or odds ratios, but directional magnitudes visible.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "18 bugs selected from 42 due to reproducibility/criteria constraints, but no power analysis or sample size justification. Single failure case (1 of 18 never repaired) reduces effective n further.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Generated 5 outputs per prompt configuration but reports 'at least one successful' binary outcome, not variance in success across runs. No std dev or variance metrics.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Baselines are ablations: S (static), S+D (static+dynamic), S+M (static+mutation), S+D+M (full). Guo et al. and HornBro results cited but not directly compared on same bugs.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Ablations of own approach are contemporary. Prior work baselines (Guo et al. 17% on quantum, HornBro 249 gates) are from recent papers (2024-2025) but not directly retested.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Systematic ablation: S → S+D (adds dynamic) → S+M (adds mutation) → S+D+M (full). Results show S+D+M best at 94.4%, isolating contribution of each component.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "RQ1: repair success rate. RQ2: correctness/completeness/complexity across position/cause/change (9 metrics). Covers repair and explanation hypotheses.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Two authors independently evaluated 72 LLM-generated explanations (9 criteria each, 648 total judgments) for correctness, completeness, complexity. Cohen's κ = 0.48 (moderate agreement).",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Bugs4Q provides test scripts for each bug. Generated repairs evaluated against held-out test suites. Results on test set, not training set.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table I breaks down by bug type (WO vs TE). Table II breaks down by explanation element (position/cause/change) and criterion (correctness/completeness/complexity).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "One program (1 of 18) could not be repaired by any configuration. Figure 2 shows this. Discussed: S+D+M fixed 17/18, unique successful repairs isolated.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Reports S+M worse than S+D (83.3% vs 88.9%), with explanation provided. For explanations, S+D+M worse at cause element than S. Cohen's κ = 0.48 moderate, not high.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Only stated 'GPT-5 via OpenAI API with default settings'. No model snapshot date, parameter count, or specific version. 'Default settings' is vague for reproducibility.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "All prompts stated to be 'available in our replication package [29]'. System prompt shared across all configs, specific content of each prompt (S/D/M components) described in Section III.B.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "Only 'default settings' for GPT-5. No temperature, top-p, max_tokens, stop sequences, or other API parameters specified.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Prompt construction detailed: static info (code + description + expected behavior), dynamic info (execution results), mutation analysis (25 operators, 4 status types, line/operator/traceback per mutation).",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Static info manually collected from source URLs (consensus reached on disagreements). Bugs reproduced before inclusion. Mutation analysis applied with QMutPy [15]. Preprocessing steps documented.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Bugs4Q publicly available with test scripts. QMutPy outputs deterministic. 360 generated repairs promised in replication package [29] with DOI.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Bugs4Q: cloned from GitHub, confirmed reproducible. Static info: two authors inspected source URLs, extracted from GitHub/Stack Overflow/Stack Exchange. Disagreements resolved by consensus.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "Not a human subjects study. Uses existing Bugs4Q dataset of real-world bugs. Recruitment N/A.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Overall pipeline shown in Figure 1: Bugs4Q → QMutPy → 4 prompt configs → GPT-5 → evaluation. Described in Section III with data types and formats.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "GPT-5 training data cutoff not stated. Paper uses GPT-5 released Nov 2025 (reference 22). Training cutoff likely mid-2024 but not explicitly provided.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether Bugs4Q bugs (from GitHub/StackOverflow) overlap with GPT-5 training data. No mitigation for potential contamination.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Bugs4Q is real-world (GitHub/Stack Overflow), created before GPT-5 training. No explicit discussion of whether examples appeared in training data.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "Not a human subjects study. No human participants. N/A.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "Not a human subjects study. No IRB approval needed. N/A.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "Not a human subjects study. No participant demographics. N/A.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "Not a human subjects study. For bug selection: included if reproducible and had mutants + accessible repo. N/A for human study criteria.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "Not a human subjects study. No randomization of participants. N/A.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "Not a human subjects study. For explanation evaluation, two authors evaluated without blinding to prompt config. Not applicable category.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "Not a human subjects study. No attrition to report. N/A.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost, API cost per repair, or latency reported. Running 360 repair attempts on GPT-5 API would have substantial cost, not disclosed.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget, wall-clock time, or resource usage (GPUs, API calls, cost) reported.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Mutation analysis results improve repair success rate of buggy quantum programs",
       "evidence": "Table I: S+D+M (with mutation) achieves 94.4% vs S (static only) 77.8%. For WO bugs: S+D+M 100% vs S+D 90%.",
       "supported": "strong"
     },
     {
       "claim": "Combining static, dynamic, and mutation information yields best repair rate",
       "evidence": "S+D+M outperforms all other configurations in Table I: 94.4% total, 100% on WO, 87.5% on TE.",
       "supported": "strong"
     },
     {
       "claim": "Mutation analysis improves quality of LLM-generated explanations",
       "evidence": "Table II: S+D+M achieves best scores in 6 of 9 evaluation items (correctness/completeness/complexity), especially for position element.",
       "supported": "strong"
     },
     {
       "claim": "Dynamic information and mutation results are not helpful for TE (exception-throwing) bugs",
       "evidence": "Table I, TE column: all configurations achieve 87.5% (7 of 8). Static information alone sufficient because error is obvious.",
       "supported": "strong"
     },
     {
       "claim": "S+M underperforms S+D because mutation analysis requires running tests, making D available",
       "evidence": "Section IV.A: S+M 83.3% < S+D 88.9%. Explained: 'availability of M implies D also available, thus lower success of S+M not concerning.'",
       "supported": "moderate"
     },
     {
       "claim": "Mutation analysis particularly effective at positional descriptions in explanations",
       "evidence": "Table II: for position element, S+D+M achieves best correctness (14/18), completeness (15/18), complexity (1/18, best).",
       "supported": "strong"
     },
     {
       "claim": "Mutation analysis less effective for explaining cause of bugs",
       "evidence": "Table II: for cause element, S achieves best correctness (14/18) and completeness (18/18), not S+D+M.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-based approach is preferable to synthesis-based for quantum APR due to flexibility and minimal gate insertion",
       "evidence": "Section III.C: LLM handles wider range of bugs (API + gate), avoids excessive gate counts (HornBro added 249 gates), provides explanations.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Incorporating mutation analysis results into prompts for LLM-based automated quantum program repair significantly improves success rates from 77.8% (static only) to 94.4% (static + dynamic + mutation), with 100% success on Wrong Output bugs. The approach also enhances explanation quality, particularly for accurately describing bug locations, though static information remains more valuable for explaining root causes.",
   "red_flags": [
     {
       "flag": "Small sample size",
       "detail": "Only 18 bugs from 42-bug benchmark. One bug unfixed by all approaches. Statistical power is limited for detecting true differences."
     },
     {
       "flag": "No significance testing",
       "detail": "Differences between configurations (77.8% → 94.4%) not tested for statistical significance. Results could be within noise."
     },
     {
       "flag": "Simulator-only evaluation",
       "detail": "All experiments on Qiskit simulator, not real quantum hardware. External validity to actual quantum computers unclear."
     },
     {
       "flag": "Single LLM tested",
       "detail": "Only GPT-5 used. Generalizability to other LLMs (Claude, Llama, open-source) unknown."
     },
     {
       "flag": "Hyperparameters vague",
       "detail": "GPT-5 'default settings' not specified. Temperature, top-p, max_tokens, stop sequences not documented. Reproducibility affected."
     },
     {
       "flag": "Moderate inter-rater agreement",
       "detail": "Cohen's κ = 0.48 for explanation evaluation. Authors acknowledge 'some subjective judgment unavoidable.' Not strong reliability."
     },
     {
       "flag": "No external baselines",
       "detail": "Only compared prompt configurations (ablations) of own approach. Did not re-run Guo et al. or HornBro on same bugs for direct comparison."
     },
     {
       "flag": "Training data contamination unaddressed",
       "detail": "GPT-5 training cutoff not stated. Bugs4Q from GitHub/StackOverflow may have overlap with training data."
     },
     {
       "flag": "TE bug success ceiling",
       "detail": "Throw Exception bugs plateau at 87.5% across all configs. Mutation info doesn't help error-throwing bugs; static info sufficient."
     },
     {
       "flag": "Causality inference limited",
       "detail": "Ablation design is good for isolating components but doesn't explain WHY mutation information helps the LLM mechanistically."
     }
   ],
   "cited_papers": [
     {
       "title": "On repairing quantum programs using chatgpt",
       "relevance": "Directly competitive baseline: Guo et al. achieved only 17% repair on quantum bugs, motivating the current LLM-based approach."
     },
     {
       "title": "Automatic repair of quantum programs via unitary operation",
       "relevance": "UnitAR synthesis-based repair. Paper contrasts LLM flexibility vs synthesis limitations (gate-only, excessive complexity)."
     },
     {
       "title": "HornBro: Homotopy-like method for automated quantum program repair",
       "relevance": "SOTA synthesis method achieving higher success than ChatGPT but adds 249 gates, reducing maintainability. Paper's key motivation."
     },
     {
       "title": "Mutation testing of quantum programs: A case study with qiskit",
       "relevance": "Core methodology paper defining 25 quantum + classical mutation operators used in this study (QMutPy framework)."
     },
     {
       "title": "A comprehensive study of bug fixes in quantum programs",
       "relevance": "Identifies common quantum bug types (API, gate-related) that the LLM approach claims to handle broadly."
     },
     {
       "title": "InferFix: End-to-end program repair with llms",
       "relevance": "Classical APR baseline showing context in prompts improves LLM repair. Paper adapts this pattern to quantum domain."
     },
     {
       "title": "A survey of learning-based automated program repair",
       "relevance": "Comprehensive review of LLM-based APR for classical programs, positioning quantum APR as underexplored."
     },
     {
       "title": "Bugs4Q: A benchmark of existing bugs for quantum program testing and debugging",
       "relevance": "The experimental benchmark used: 42 real-world quantum bugs from GitHub/StackOverflow with test suites."
     },
     {
       "title": "Evaluating mutation-based fault localization for quantum programs",
       "relevance": "Recent work (2025) showing mutation analysis effective for quantum fault localization, supporting its use here."
     },
     {
       "title": "Quantum software engineering: Roadmap and challenges ahead",
       "relevance": "Broad context: quantum software engineering challenges, technical debt, code smells motivating APR tooling."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Useful for Qiskit developers but only on 18 tested bugs, requires expensive GPT-5 API access, simulator-only tested. Moderate practical applicability."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that more context helps LLM performance is expected. Mutation analysis improving prompt quality is incremental rather than surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety concerns. Automated program repair for quantum computing is not a safety-sensitive application."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no conflicting claims, no social/ethical angle. Technical contribution without drama."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Requires Bugs4Q, Qiskit, QMutPy, GPT-5 API access. Reproducible from replication package but not trivially runnable."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Authors from reputable Japanese universities (Osaka, Kyushu). Uses GPT-5 (OpenAI brand). Not top-tier AI lab but solid institutions."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "46837037",
         "title": "Proc3D: Procedural 3D Generation and Parametric Editing of 3D Shapes with LLMs",
         "points": 5,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46837037"
       },
       {
         "hn_id": "46859436",
         "title": "Hybrid Concolic Testing with Large Language Models for Guided Path Exploration",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46859436"
       }
     ],
     "top_points": 5,
     "total_points": 6,
     "total_comments": 0
   }
 }