ai-research-survey

scan.json (23689B)

---

 {
   "paper": {
     "title": "Advancing Software Quality: A Standards-Focused Review of LLM-Based Assurance Techniques",
     "authors": [
       "Avinash Patil"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2505.13766"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, Zenodo archive, or any code repository URL is provided. This survey could have released analysis scripts or data extraction tools, but none are mentioned."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The survey's corpus of 223+ reviewed papers is not released as a structured dataset. No download link or supplementary data file is provided."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature survey with no computational experiments, so there is no software environment to specify."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions are provided for how to replicate the literature search, paper selection, or analysis. A reader cannot reproduce the paper corpus without knowing what databases were searched and what queries were used."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive literature survey, not a meta-analysis. It reports raw counts and percentages of papers but does not aggregate effect sizes or statistics that would require confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative statistical claims are made that require significance testing. The paper reports descriptive counts from the corpus."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive survey with no meta-analytic aggregation, so effect sizes are not applicable."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or statistical analyses were conducted; this is a literature survey."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs were conducted; this is a literature survey reporting descriptive statistics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare itself against prior surveys in the LLM-SQA space. No prior reviews are systematically compared to demonstrate additive value or methodological improvement."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Since baselines_included applies (prior surveys in LLM-SQA exist and could be compared against), the contemporariness of those baselines also applies. However, since the paper includes no baseline comparisons at all, it cannot be assessed whether baselines are contemporary. Answer is false because no baselines are used."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; ablation studies do not apply."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The survey characterizes the literature using multiple dimensions: publication trends, dataset utilization (Figure 2), evaluation approaches (Figure 3), fine-tuning adoption (Figure 4), LLM usage distribution (Figure 5), and prompting strategies (Figure 6)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of the survey's outputs (e.g., mapping quality, completeness of the standards analysis) is not conducted and is not clearly required for this type of review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; there is no train/test split applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides a detailed breakdown of LLM applications mapped to specific standards (ISO/IEC 12207, 25010, 5055, ISO 9001, CMMI, TMM) with per-category listings of applications and references."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI (Challenges, Limitations, and Risks) discusses failure modes and limitations of LLM-based SQA, including data privacy exposure, model bias, black-box explainability problems, and resource constraints."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V.B notes that 'nearly 30% of the reviewed papers did not specify any dataset,' Section V.E notes that 'nearly 19% of papers did not specify which LLM was used,' and Section VI identifies challenges and failure patterns in the literature."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract promises a survey of LLM-SQA intersection, discussion of challenges, and future directions. The paper delivers on these promises with sections covering each claim, including empirical case studies mentioned and standards mapping in Table I."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims itself; it surveys and categorizes existing work without making causal assertions about what causes quality improvements."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about LLM potential for SQA (e.g., 'LLMs can be integrated into SQA in a trustworthy, efficient, and standards-aligned manner') without adequately bounding these to the specific papers reviewed. The survey covers 2023-2025 literature yet presents conclusions that generalize broadly without acknowledging the novelty and instability of the field."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive survey that does not present original empirical results. The NA rule applies: the paper makes no empirical claims that would require alternative explanations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments with LLMs, so model versions are not applicable."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No LLM prompting is used in this survey paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLM experiments are conducted; this is a literature survey."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; this is a literature survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section IV describes 7 high-level selection criteria for including papers (focus on LLM-SQA, alignment with standards, empirical rigor, recency, breadth, practical relevance, peer review). However, it does not describe the actual search process: which databases were queried, what search strings were used, how many papers were initially found, or how many were excluded at each stage. 'Over 223 papers' are mentioned without explaining how the starting pool was identified or how it was filtered."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section VI is titled 'Challenges, Limitations, and Risks' but addresses limitations of LLM technology adoption in general, not limitations of this survey's methodology. There is no substantive discussion of the survey's own limitations (e.g., possible incomplete coverage, possible publication bias, lack of quality assessment of reviewed papers, single-author review without inter-rater reliability)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity specific to this survey are discussed. The paper does not acknowledge threats such as: selection bias from the informal search process, the lack of formal systematic review protocol, single-author classification without reliability checks, or possible incomplete coverage of the 2023-2025 literature."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not explicitly state what it does NOT cover. For example, it does not clarify whether non-English papers were excluded, which databases were searched, whether gray literature was included, or what LLM tasks are out of scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The full list of 223+ reviewed papers is not provided as a structured dataset. Readers cannot verify which papers were included in each category or reproduce the classification decisions."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Section IV lists inclusion criteria but does not describe the actual data collection procedure: no search databases are named (e.g., IEEE Xplore, ACM DL, Semantic Scholar), no search queries are given, and no time period for the search itself is stated beyond 'papers published from 2023 onward.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; this is a literature survey using published papers as data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from initial paper discovery to the final set of 223+ papers is not documented. There is no PRISMA-style flowchart or equivalent showing how many papers were initially found, how many were excluded at each stage, and why. The reader is left with 7 inclusion criteria but no evidence of how they were applied."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present in the paper. The author's affiliation with Juniper Networks Inc. is listed but no statement about whether Juniper funded or supported this work is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation with Juniper Networks Inc. is disclosed on the title page alongside ORCID. The paper does not evaluate Juniper products specifically, so this affiliation does not constitute a direct conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The author is affiliated with Juniper Networks Inc. but no funding source is disclosed. Absence of a funding disclosure is not evidence of being unfunded — the work could have been done on company time. Since we cannot verify funder independence, answer is false."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests is provided anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature survey that does not evaluate a pre-trained model on any benchmark. Contamination questions do not apply."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of pre-trained models is conducted in this paper."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of pre-trained models is conducted in this paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no original LLM experiments, so inference cost is not applicable."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no original computational experiments, so compute budget is not applicable."
       }
     }
   },
   "claims": [
     {
       "claim": "The number of publications on LLMs in SQA increased sharply after 2023: 31 papers in 2023, 91 in 2024, and 127 in 2025.",
       "evidence": "Section V.A, Figure 1. Counts are stated in the text.",
       "supported": "weak"
     },
     {
       "claim": "Nearly 30% of the reviewed papers did not specify any dataset, highlighting a concerning gap in reproducibility.",
       "evidence": "Section V.B, Figure 2. Percentage stated without showing the exact count or how papers were classified.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 was the most commonly used model (21%), and nearly 19% of papers did not specify which LLM was used.",
       "evidence": "Section V.E, Figure 5. Percentages stated without showing raw counts or the classification methodology.",
       "supported": "moderate"
     },
     {
       "claim": "Only 14.3% of the studies reported using fine-tuned models; most leveraged zero-shot or few-shot prompting.",
       "evidence": "Section V.D, Figure 4.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs can be integrated into SQA processes across all major software quality standards (ISO/IEC 12207, 25010, 5055, ISO 9001, CMMI, TMM) in a trustworthy and standards-aligned manner.",
       "evidence": "Table I provides a mapping of LLM applications to standards attributes, citing 200+ papers across categories. However, the survey makes no original experiments to validate these mappings.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "This survey of 223+ papers (2023-2025) finds that LLM-based techniques are being applied across all major software quality assurance domains including requirement validation, defect detection, test generation, and compliance checks. Publication volume has grown nearly fourfold from 2023 to 2025. GPT-4 dominates usage (21%), while 19% of papers failed to specify the LLM used and 30% did not specify their dataset, indicating widespread reproducibility gaps in the field. The paper maps LLM applications to six major software quality standards (ISO/IEC 12207, 25010, 5055, ISO 9001, CMMI, TMM) and identifies governance, explainability, and data privacy as key outstanding challenges.",
   "red_flags": [
     {
       "flag": "Informal search methodology without PRISMA or systematic review protocol",
       "detail": "Section IV lists 7 inclusion criteria but never describes which databases were searched, what search queries were used, how many papers were initially retrieved, or how papers were filtered to reach the final set of 223+. This is not a systematic review in any rigorous sense."
     },
     {
       "flag": "Single-author review without inter-rater reliability",
       "detail": "The entire paper selection and categorization was apparently performed by a single author (Avinash Patil) with no mention of a second reviewer or inter-rater reliability checks. Systematic reviews typically require at least two independent reviewers to control for selection and classification bias."
     },
     {
       "flag": "No quality assessment of reviewed papers",
       "detail": "The survey classifies papers by topic and counts them but applies no quality filter to the 223+ papers beyond the seven informal inclusion criteria. Weak or methodologically flawed papers are treated equally with rigorous ones, which launders their results into the survey's conclusions."
     },
     {
       "flag": "Broad unsupported generalizations from narrow evidence",
       "detail": "The abstract and conclusion claim LLMs can be integrated 'in a trustworthy, efficient, and standards-aligned manner' but the survey itself presents evidence that most studies lack reproducibility (30% no dataset, 19% no LLM specification). This contradicts the optimistic framing."
     },
     {
       "flag": "No competing interests or funding disclosure",
       "detail": "The paper lists affiliation with Juniper Networks Inc. but provides no statement on whether the research was funded, whether there are competing interests, or whether Juniper supported this work."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": [
         "M. Chen",
         "J. Tworek",
         "H. Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational paper on evaluating LLMs for code (Codex/GitHub Copilot), directly relevant to LLM coding capability assessment."
     },
     {
       "title": "LLM-based test-driven interactive code generation: User study and empirical evaluation",
       "authors": [
         "S. Fakhoury",
         "A. Naik",
         "G. Sakkas",
         "S. Chakraborty",
         "S. K. Lahiri"
       ],
       "year": 2024,
       "relevance": "Empirical user study of LLM-based code generation combining test-driven development, directly relevant to LLM capability evaluation."
     },
     {
       "title": "Mutation-guided LLM-based test generation at Meta",
       "authors": [
         "C. Foster",
         "A. Gulati",
         "M. Harman"
       ],
       "year": 2025,
       "arxiv_id": "2501.12862",
       "relevance": "Industrial-scale study of LLM-based test generation at Meta, relevant to real-world LLM deployment in software engineering."
     },
     {
       "title": "AI-powered code review with LLMs: Early results",
       "authors": [
         "Z. Rasheed",
         "M. A. Sami",
         "M. Waseem"
       ],
       "year": 2024,
       "arxiv_id": "2404.18496",
       "relevance": "Empirical evaluation of LLM-based automated code review, directly relevant to LLM software quality applications."
     },
     {
       "title": "Advancing requirements engineering through generative AI: Assessing the role of LLMs",
       "authors": [
         "C. Arora",
         "J. Grundy",
         "M. Abdelrazek"
       ],
       "year": 2024,
       "relevance": "Study of LLMs for requirements engineering including practitioner feedback, relevant to LLM-assisted software development evaluation."
     },
     {
       "title": "Fine-tuning and prompt engineering for large language models-based code review automation",
       "authors": [
         "C. Pornprasit",
         "C. Tantithamthavorn"
       ],
       "year": 2024,
       "relevance": "Empirical comparison of fine-tuning vs. prompt engineering for LLM code review automation, directly relevant to LLM methodology evaluation."
     },
     {
       "title": "Large language models as configuration validators",
       "authors": [
         "X. Lian",
         "Y. Chen",
         "R. Cheng"
       ],
       "year": 2025,
       "relevance": "Evaluation of LLMs for software configuration validation, relevant to LLM software quality assurance applications."
     },
     {
       "title": "LLM-assisted static analysis for detecting security vulnerabilities",
       "authors": [
         "Z. Li",
         "S. Dutta",
         "M. Naik"
       ],
       "year": 2024,
       "arxiv_id": "2405.17238",
       "relevance": "Study of LLMs for security vulnerability detection via static analysis, directly relevant to LLM-based SQA."
     },
     {
       "title": "Enhancing large language models for text-to-testcase generation",
       "authors": [
         "S. Alagarsamy",
         "C. Tantithamthavorn",
         "C. Arora",
         "A. Aleti"
       ],
       "year": 2024,
       "arxiv_id": "2402.11910",
       "relevance": "Empirical study of LLMs for automated test case generation, relevant to LLM test automation capability assessment."
     },
     {
       "title": "An empirical study on the code refactoring capability of large language models",
       "authors": [
         "J. Cordeiro",
         "S. Noei",
         "Y. Zou"
       ],
       "year": 2024,
       "arxiv_id": "2411.02320",
       "relevance": "Empirical evaluation of LLM refactoring capability, directly relevant to LLM software quality assurance assessment."
     },
     {
       "title": "Expectations vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": [
         "P. Vaithilingam"
       ],
       "year": 2022,
       "relevance": "CHI user study on the usability of LLM-based code generation tools, relevant to human factors in LLM-assisted development."
     },
     {
       "title": "Industry-academia collaborations in software testing: experience and success stories from Canada",
       "authors": [
         "V. Garousi",
         "K. Petersen",
         "B. Ozkan"
       ],
       "year": 2019,
       "arxiv_id": "1904.04986",
       "relevance": "Empirical study of software testing practices, relevant to understanding software quality assessment methodology."
     }
   ]
 }