ai-research-survey

scan.json (23035B)

---

 {
   "paper": {
     "title": "Large Language Models for Requirements Engineering: A Systematic Literature Review",
     "authors": ["Mohammad Amin Zadenoori", "Jacek Dąbrowski", "Waad Alhoshan", "Liping Zhao", "Alessio Ferrari"],
     "year": 2025,
     "venue": "arXiv (preprint submitted to Elsevier)",
     "arxiv_id": "2509.11446"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This SLR of 74 papers (2023-2024) on LLMs for Requirements Engineering finds a 136% growth from 2023 to 2024, with research shifting from traditional NLP4RE tasks (defect detection, classification) toward cognitively demanding tasks like elicitation and validation (20% each). GPT-family models dominate (77%), Zero-shot (38%) and Few-shot (26%) prompting dominate, while RAG (7%) and interactive prompting (4%) remain underexplored. Evaluation relies heavily on laboratory experiments (76%) with limited real-world validation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The authors state 'The spreadsheets resulting from our data extraction and grouping, along with all synthesized data, including generated statistics and classifications, are publicly available in the supplementary material of this survey [13]' with a Zenodo link (doi:10.5281/zenodo.17068810)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Data extraction forms, classification spreadsheets, and synthesized data are shared via Zenodo [13]."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. This is a survey paper but analysis scripts (if any) have no documented environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided for replicating the search, screening, or data extraction process beyond what is described in the methodology section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that reports descriptive statistics (counts, percentages) of the literature. No experimental results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper reporting descriptive statistics only; no comparative claims requiring significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental comparisons requiring effect sizes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; sample size is determined by the systematic search process, not statistical requirements."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental runs to report variance across."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The survey explicitly compares its findings against the prior NLP4RE survey by Zhao et al. (2021) covering 404 studies from 1983-2019, noting shifts in task focus, artifact types, and phase distribution."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Zhao et al. (2021) is the most relevant prior survey in NLP4RE. The paper also discusses related SLRs by Hou et al. (2024), Wang et al. (2024), and others in Section 3."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no system performance metrics to evaluate."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no system outputs to evaluate with human judges."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no train/test split concept applies."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down extensively by RE task (Table 5), RE phase (Table 6), input artifact (Table 7), output artifact (Table 8), prompting strategy (Table 10), LLM type (Table 14), evaluation method (Tables 16-17), and more."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies underexplored areas (Requirements Retrieval, Terminology Extraction at 1% each), gaps in reproducibility (only 16% share datasets), lack of field studies (0% field experiments), and over-reliance on GPT models and laboratory experiments."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings: 39% of studies provide no supplementary materials, 32% don't specify prompt selection strategy, 0% use field experiments or sample studies, and only 16% share datasets."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 74 studies, 136% growth, Zero-shot (44%→38% in body) and Few-shot (29%→26% in body) dominance, RAG (6%→7%) and Interactive (5%→4%) being underexplored are all supported by tables in Section 5. Note slight discrepancies between abstract and body percentages (e.g., '44%' in abstract vs '38%' in body for Zero-shot) but these appear to be due to multi-label counting differences."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-style claims like 'LLMs are not merely automating traditional NLP4RE tasks but are enabling research to tackle tasks that were previously considered difficult for machines' and 'RE researchers, like the general public, engaged with LLMs once easy-to-use interfaces lowered technical barriers.' These causal attributions are not tested — the paper only observes correlations in publication patterns."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds to 74 studies from 2023-2024, acknowledges the time boundary of its search (September/December 2024), and limits claims to the reviewed corpus. Section 7 discusses threats to validity."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the shift from NLP4RE tasks to LLM4RE tasks could be due to publication venue selection bias, researcher incentives for novelty, or the specific search string used — none of these are explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses publication counts and categorization frequencies as proxies for research trends and community interest but does not discuss limitations of this proxy (e.g., publication counts don't reflect research quality or actual adoption)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use any LLM or model for its own analysis."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no model-based experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows the full search pipeline with counts at each stage: 244 from primary search → 136 after screening → 62 after criteria; 940 from secondary → 887 after dedup → 120 after screening → 12 after criteria. Table 2 provides specific inclusion/exclusion criteria. The search string is provided verbatim."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides a dedicated discussion of study selection validity, data validity, and research validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: single database (Scopus), data extraction performed by first author only without systematic cross-checking, subjectivity of classification schemes, and plans for future cross-validation. The paper also includes a disclaimer that 'No systematic cross-checking has been performed.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly scopes to generative pre-trained transformers (not encoder-only like BERT), 2023-2024 time period, peer-reviewed studies in CORE A*/A/B conferences and Q1/Q2 journals. States it is 'a preliminary study' in the disclaimer."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The data extraction spreadsheets are available via Zenodo [13] (doi:10.5281/zenodo.17068810), allowing independent verification of classifications."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the search strategy in detail: search string provided verbatim, Scopus database, execution date (September 11, 2024), secondary venue-based search (December 18, 2024), specific venues listed in Table 1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is a literature search of published papers."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 documents each stage with counts: primary search (244 → 136 → 62), secondary search (940 → 887 → 120 → 12), yielding 74 total. Table 2 provides inclusion/exclusion criteria. Section 4.3 describes the data extraction form and classification scheme construction."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Padova, Lero/University of Limerick, IMSIU, University of Manchester, University College Dublin."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed. One author is at Lero (Research Ireland Centre for Software) which may have funding, but this is not stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no inference or computational method to report costs for."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The paper follows Kitchenham's SLR guidelines (Section 4), includes a structured search process with a flow diagram (Figure 2), defined search strings, inclusion/exclusion criteria (Table 2), and a two-stage search strategy (primary digital library + secondary venue-based search)."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its 74 primary studies. It categorizes them by task, phase, prompting strategy, etc., but treats all papers equally regardless of quality. Table 2 uses venue quality (CORE ranking, SJR quartile) as an inclusion filter, but this is access control, not quality assessment of the studies themselves."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss publication bias. It does not consider whether the 74 studies skew toward positive results about LLMs for RE, nor does it discuss the absence of negative-result papers in the corpus."
       }
     }
   },
   "claims": [
     {
       "claim": "74 primary studies on LLMs for RE were published between 2023-2024, with a 136% increase from 2023 to 2024.",
       "evidence": "Section 5.1: 22 studies in 2023, 52 in 2024, shown in Figure 3.",
       "supported": "strong"
     },
     {
       "claim": "LLM4RE shifts focus from traditional NLP4RE tasks (defect detection, classification) to cognitively demanding tasks (elicitation, validation at 20% each).",
       "evidence": "Section 5.2.1, Table 5 shows distribution. Comparison with Zhao et al. (2021) discussed in text, noting defect detection and classification were dominant in NLP4RE but rank 8th and 5th in LLM4RE.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-family models dominate (77% of studies), with limited exploration of open-source alternatives.",
       "evidence": "Section 5.4.3, Table 14: GPT in 57/74 studies (77%), LLaMA in 9 (12%), Mixtral/Mistral in 3 (4%).",
       "supported": "strong"
     },
     {
       "claim": "Zero-shot (38%) and Few-shot (26%) prompting dominate, while RAG (7%) and Interactive (4%) remain underexplored.",
       "evidence": "Section 5.3.1, Table 10 shows the distribution of prompting strategies across studies.",
       "supported": "strong"
     },
     {
       "claim": "Laboratory experiments dominate evaluation (76%), with limited real-world validation (Field Studies 7%, Field Experiments 0%).",
       "evidence": "Section 5.6.2, Table 17 shows 56/74 studies use laboratory experiments, 5 use field studies, 0 use field experiments.",
       "supported": "strong"
     },
     {
       "claim": "Only 16% of studies shared publicly available datasets, creating replicability problems.",
       "evidence": "Section 5.4.1: 'only 16% of the studies made use of public datasets' with 10 datasets identified in Table 13.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of source papers",
       "detail": "The survey categorizes and counts 74 papers but does not assess their methodological quality. All papers are treated equally regardless of rigor, potentially laundering weak results alongside strong ones."
     },
     {
       "flag": "Single-author data extraction without systematic cross-checking",
       "detail": "The paper explicitly acknowledges that data extraction was performed by the first author only, with the last author consulted 'in case of doubts.' The disclaimer states 'No systematic cross-checking has been performed.' This introduces potential bias in classification decisions."
     },
     {
       "flag": "Percentage discrepancies between abstract and body",
       "detail": "The abstract reports Zero-shot at 44% and Few-shot at 29%, while Section 5.3.1 reports 38% and 26%. This appears due to different denominators (per-study vs per-technique) but is not explained, creating confusion."
     },
     {
       "flag": "Self-labeled as preliminary",
       "detail": "The paper includes a disclaimer: 'This paper is a preliminary study. No systematic cross-checking has been performed on the retrieved and analysed studies.' This is unusually candid but raises questions about whether the findings are reliable enough for citation."
     },
     {
       "flag": "No publication bias consideration",
       "detail": "The survey does not discuss whether its corpus of 74 papers is biased toward positive results about LLMs for RE. Given that negative results are underrepresented in CS venues generally, the optimistic framing of LLM4RE could reflect publication bias."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["A. Fan", "B. Gokkaya", "M. Harman", "M. Lyubarskiy", "S. Sengupta", "S. Yoo", "J. M. Zhang"],
       "year": 2023,
       "doi": "10.1109/ICSE-FoSE59343.2023.00008",
       "relevance": "Foundational survey on LLMs for SE, highlighting emergent properties and hybrid approaches."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu", "Z. Yang", "K. Wang", "L. Li", "X. Luo", "D. Lo", "J. Grundy", "H. Wang"],
       "year": 2024,
       "doi": "10.1145/3695988",
       "relevance": "SLR of 395 publications on LLMs in SE (2017-2024), covering data collection, optimization, and evaluation."
     },
     {
       "title": "LLMs and LLM-based agents in software engineering: A survey",
       "authors": ["M. Jin", "S. Kumar", "W. Zhang"],
       "year": 2024,
       "doi": "10.1016/j.jss.2023.111589",
       "relevance": "Survey of LLM-based agents with tools, memory, and decision-making for SE tasks."
     },
     {
       "title": "LLM-based multi-agent systems for software engineering: Literature review, vision, and the road ahead",
       "authors": ["J. He", "C. Treude", "D. Lo"],
       "year": 2025,
       "doi": "10.1145/3712003",
       "relevance": "Review of multi-agent LLM systems for SE with framework capabilities and limitations."
     },
     {
       "title": "Software testing with large language models: Survey, landscape, and vision",
       "authors": ["J. Wang", "Y. Huang", "C. Chen", "Z. Liu", "S. Wang", "Q. Wang"],
       "year": 2024,
       "doi": "10.1109/TSE.2024.3368208",
       "relevance": "Survey of 102 studies on LLMs for software testing including prompt engineering and scalability challenges."
     },
     {
       "title": "Natural language processing for requirements engineering: A systematic mapping study",
       "authors": ["L. Zhao", "W. Alhoshan", "A. Ferrari", "K. J. Letsholo", "M. A. Ajagbe", "E.-V. Chioasca", "R. T. Batista-Navarro"],
       "year": 2021,
       "doi": "10.1145/3444689",
       "relevance": "Predecessor survey of 404 NLP4RE studies (1983-2019), used as the primary comparison baseline."
     },
     {
       "title": "The prompt report: a systematic survey of prompt engineering techniques",
       "authors": ["S. Schulhoff"],
       "year": 2024,
       "arxiv_id": "2406.06608",
       "relevance": "Comprehensive survey of prompting techniques, cited as a map for exploration in RE prompt engineering."
     },
     {
       "title": "Evaluation guidelines for empirical studies in software engineering involving LLMs",
       "authors": ["S. Baltes"],
       "year": 2025,
       "arxiv_id": "2508.15503",
       "relevance": "Framework for empirical LLM studies in SE with focus on replicability and verifiability."
     },
     {
       "title": "ClarifyGPT: A framework for enhancing LLM-based code generation via requirements clarification",
       "authors": ["F. Mu", "L. Shi", "S. Wang", "Z. Yu", "B. Zhang", "C. Wang", "S. Liu", "Q. Wang"],
       "year": 2024,
       "relevance": "LLM-based tool for requirements clarification in code generation, bridging RE and SE."
     },
     {
       "title": "Software architecture meets LLMs: A systematic literature review",
       "authors": ["L. Schmid", "T. Hey", "M. Armbruster", "S. Corallo", "D. Fuchß", "J. Keim", "H. Liu", "A. Koziolek"],
       "year": 2025,
       "arxiv_id": "2505.16697",
       "relevance": "SLR on LLMs for software architecture tasks, complementary domain-specific survey."
     }
   ]
 }