ai-research-survey

scan-v5.json (20463B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "Large Language Models for Requirements Engineering: A Systematic Literature Review",
     "authors": [
       "Mohammad Amin Zadenoori",
       "Jacek Dąbrowski",
       "Waad Alhoshan",
       "Liping Zhao",
       "Alessio Ferrari"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.11446",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and Section 1 state 'Zero-shot (44%) and Few-shot (29%)' but Table 10 shows 38% and 26% respectively; Section 5.4.2 reports 48% share code/prompts while Section 6.5 claims 61% — these internal inconsistencies undermine the abstract's accuracy.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims LLM4RE growth was 'catalyzed by the availability of accessible interfaces' (ChatGPT caused the surge) but this is a correlation-based observation from a literature count; no causal design supports it.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally scoped to 'the reviewed 74 studies' and the 2023–2024 window; broad statements about the field are framed as observations from the corpus, not universal claims.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider that dominance of elicitation/validation tasks may reflect venue bias (74% of papers appear at RE conference and its workshops), nor does it address selection effects from using only Scopus.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures paper counts, task frequencies, and strategy distributions as direct descriptions of the literature corpus rather than claiming they proxy external outcomes.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Threats to Validity') is a dedicated section addressing study selection validity, data validity, and research validity.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: single-database limitation (Scopus only), single-author data extraction without systematic cross-checking, and reliance on venue ranking as a quality proxy — all named explicitly.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Scope is bounded to 2023–2024 peer-reviewed studies in CORE A*/A/B conferences and Q1/Q2 journals found via Scopus plus manual venue search; the paper also includes an explicit preliminary-study disclaimer.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment appears anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors list institutional affiliations (Padova, Limerick, IMSIU, Manchester, UCD) on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder is disclosed, making independence assessment inapplicable; however, first author Zadenoori is cited as [117] and co-author Ferrari's work is cited as [66, 88] without disclosure of this self-citation bias.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 formally defines Requirements Engineering (phases, tasks, artifacts) and LLMs (architecture, capabilities), and Figure 1 provides an explicit conceptual scheme.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it provides a systematic mapping of 74 LLM4RE studies across demographics, RE tasks, prompting strategies, resources, and evaluation methods, structured around five research questions.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 discusses multiple prior surveys (Zhao et al. NLP4RE, Hou et al. LLMs in SE, Jin et al., He et al.) and the paper consistently compares its findings against Zhao et al. [27] throughout the results.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "The exact Scopus search string is provided verbatim in Section 4.2, and the secondary-search venues are listed in Table 1 with execution dates given.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 lists four inclusion criteria (I1–I3) and four exclusion criteria (E1–E4) with explicit definitions, including venue-quality thresholds via CORE and SJR rankings.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": true,
           "justification": "The paper follows Kitchenham's SLR guidelines [26] and provides a PRISMA-style flow diagram (Figure 2) with paper counts at each filtering stage.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "The full Boolean search string is quoted in Section 4.2, combining LLM-representative terms with RE-domain terms using explicit AND/OR operators.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Scopus is named as the primary database; Table 1 lists eight specific journals/conferences for secondary manual search.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Figure 2 documents the screening funnel with counts: 244 initial → 136 after title/abstract screening → 62 after inclusion/exclusion + 12 from secondary search = 74 final studies.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "The 2023–2024 scope is justified by the observation that no LLM4RE papers appeared before ChatGPT's release; Scopus is justified for its coverage of IEEE/ACM/Elsevier venues; venue selection for secondary search is explained.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": false,
           "justification": "The paper categorizes and counts findings but does not explicitly discuss cases where primary studies report conflicting results on effectiveness or applicability of LLMs to the same RE task.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "Venue ranking (CORE A*/A/B, Q1/Q2) is used as an inclusion filter, not a quality rubric applied to included studies; no risk-of-bias assessment or quality scoring of the 74 papers is performed.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss publication bias or acknowledge that the reviewed studies likely overrepresent positive findings about LLM effectiveness in RE tasks.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": false,
           "justification": "The synthesis is limited to descriptive frequency counts and percentages; there is no meta-analysis, effect size aggregation, or statistical comparison of results across studies.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Future-work recommendations in Section 6 are consistently tied to identified gaps — e.g., recommending more RAG-based approaches because only 7% of studies use them, or more Field Studies because 76% rely on lab experiments.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Publications on LLM4RE grew 136% from 2023 to 2024, with 22 studies in 2023 and 52 in 2024.",
       "evidence": "Figure 3 and Section 5.1 report exact counts with year-over-year comparison.",
       "supported": "strong"
     },
     {
       "claim": "Requirements Elicitation and Validation each account for 20% of studied tasks, dominating the LLM4RE landscape.",
       "evidence": "Table 5 documents 15 studies each (20%) for these tasks across the 74 primary studies.",
       "supported": "strong"
     },
     {
       "claim": "Zero-shot prompting is used in 38% of studies and K-shot in 26%, dominating prompt engineering strategies.",
       "evidence": "Table 10 reports counts of 28 and 19 studies respectively; however, the abstract and introduction incorrectly state 44% and 29%, creating an internal inconsistency.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-family models dominate LLM adoption, cited in 77% of reviewed studies.",
       "evidence": "Table 14 lists 57/74 studies using GPT models.",
       "supported": "strong"
     },
     {
       "claim": "Only 16% of studies use publicly available datasets, creating significant reproducibility barriers.",
       "evidence": "Section 5.4.1 identifies 10 public datasets used in 12 of 74 studies (16%).",
       "supported": "strong"
     },
     {
       "claim": "Laboratory experiments dominate evaluation (76%), with zero Field Experiments or Sample Studies.",
       "evidence": "Table 17 shows 56/74 studies use laboratory experiments and 0 use field experiments or sample studies.",
       "supported": "strong"
     },
     {
       "claim": "RAG-based and interactive prompting remain severely underexplored at 7% and 4% of studies respectively.",
       "evidence": "Table 10 reports 5 studies using RAG-driven prompting and 3 using Interactive prompting out of 74.",
       "supported": "strong"
     },
     {
       "claim": "48% of studies provide source code or prompts, indicating growing open science awareness.",
       "evidence": "Section 5.4.2 reports 36/74 studies (48%), though Section 6.5 contradictorily states '45 studies (61%)' — a direct internal inconsistency.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "This SLR of 74 LLM4RE studies (2023–2024) finds the field exploded after ChatGPT's release, growing 136% in one year and shifting focus from traditional NLP4RE tasks (defect detection, classification) to cognitively demanding work like elicitation and validation (20% each). Zero-shot and few-shot prompting dominate (38%/26%) while advanced techniques like RAG (7%) remain underexplored. GPT models account for 77% of model usage. Reproducibility is poor: only 16% of studies share public datasets, and 76% rely solely on laboratory experiments with no field experiments or sample studies at all. The paper also self-identifies as a preliminary study with no systematic inter-rater reliability checks on data extraction.",
   "red_flags": [
     {
       "flag": "Abstract/intro statistics contradict results tables",
       "detail": "Abstract and Section 1 state Zero-shot=44%, Few-shot=29%, but Table 10 shows 38% and 26% respectively. Section 5.4.2 reports 48% code sharing but Section 6.5 says 61% — the same paper contradicts itself on basic counts."
     },
     {
       "flag": "Single-author data extraction, no inter-rater reliability",
       "detail": "The paper explicitly states data extraction was performed by the first author alone with only selective cross-checking by the last author — acknowledged in Section 4.3 and the preliminary study disclaimer."
     },
     {
       "flag": "Undisclosed self-citation",
       "detail": "First author Zadenoori's own paper [117] and co-author Ferrari's papers [66, 88] are included among the reviewed studies without disclosure of potential bias in classification or interpretation."
     },
     {
       "flag": "Single database primary search",
       "detail": "Primary search uses only Scopus; ACM Digital Library, IEEE Xplore, and Google Scholar are excluded from the primary search, with coverage gaps only partially addressed by the secondary venue-based search."
     },
     {
       "flag": "No quality assessment of included studies",
       "detail": "Venue ranking is used as an inclusion filter but no study-level quality rubric is applied; a workshop paper at a B-ranked conference is treated the same as a journal article with rigorous methodology."
     },
     {
       "flag": "No publication bias discussion",
       "detail": "The paper does not acknowledge that published studies disproportionately report positive results, which could inflate the apparent capability of LLMs for RE tasks."
     }
   ],
   "cited_papers": [
     {
       "title": "Natural Language Processing for Requirements Engineering: A Systematic Mapping Study",
       "relevance": "Primary baseline comparison — Zhao et al.'s NLP4RE survey covering 1983–2019 that this paper explicitly continues and compares against throughout."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review (Hou et al., 2024)",
       "relevance": "Directly comparable SLR of 395 LLM4SE publications; examines overlapping territory across the broader SE field."
     },
     {
       "title": "The ABC of Software Engineering Research (Stol & Fitzgerald, 2018)",
       "relevance": "Classification framework for empirical strategies used throughout the paper to categorize evaluation approaches in primary studies."
     },
     {
       "title": "Guidelines for Performing Systematic Literature Reviews in Software Engineering (Kitchenham, 2011)",
       "relevance": "Methodological protocol this SLR claims to follow; foundational reference for SLR methodology in SE."
     },
     {
       "title": "Evaluation Guidelines for Empirical Studies in Software Engineering Involving LLMs (Baltes et al., 2025)",
       "relevance": "Proposed evaluation framework for LLM studies in SE; directly relevant to methodological quality assessment of the primary studies reviewed."
     },
     {
       "title": "Replication in Requirements Engineering: The NLP for RE Case (Abualhaija et al., 2024)",
       "relevance": "Addresses reproducibility challenges in NLP4RE, directly informing the paper's analysis of dataset sharing and replicability in LLM4RE."
     },
     {
       "title": "LLM-Based Multi-Agent Systems for Software Engineering (He et al., 2025)",
       "relevance": "Related SLR covering multi-agent LLM systems for SE; relevant to agentic AI methodology assessment context."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Provides a comprehensive taxonomy of tools, datasets, and prompting strategies that practitioners can immediately apply to requirements engineering workflows."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that RAG and interactive prompting are severely underexplored (7%/4%) despite being key LLM strengths is mildly contrarian; the poor reproducibility rate is notable."
     },
     "fear_safety": {
       "score": 0,
       "justification": "Briefly mentions safety-critical domains (healthcare, autonomous systems) as motivation but does not raise AI risk concerns as a central theme."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or adversarial framing; purely a mapping study presenting descriptive findings."
     },
     "demo_ability": {
       "score": 0,
       "justification": "This is a literature survey; there is nothing to demo or try directly."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from European universities without a famous lab affiliation; no industry partner involvement."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44332699",
         "title": "AbsenceBench: Language models can't tell what's missing",
         "points": 324,
         "comments": 84,
         "url": "https://news.ycombinator.com/item?id=44332699",
         "created_at": "2025-06-20T22:26:52Z"
       },
       {
         "hn_id": "44116412",
         "title": "FlowTSE: Target Speaker Extraction with Flow Matching",
         "points": 25,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=44116412",
         "created_at": "2025-05-28T14:30:33Z"
       },
       {
         "hn_id": "43451552",
         "title": "Blockchain with Proof of Quantum Work",
         "points": 5,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43451552",
         "created_at": "2025-03-23T08:24:58Z"
       },
       {
         "hn_id": "44445700",
         "title": "A nanosecond-duration radio pulse originating from the defunct Relay 2 satellite",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44445700",
         "created_at": "2025-07-02T16:31:26Z"
       },
       {
         "hn_id": "43424742",
         "title": "Blockchain with Proof of Quantum Work",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43424742",
         "created_at": "2025-03-20T15:35:28Z"
       },
       {
         "hn_id": "41630923",
         "title": "Marca: Mamba Accelerator with ReConfigurable Architecture",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41630923",
         "created_at": "2024-09-23T21:43:44Z"
       },
       {
         "hn_id": "45250763",
         "title": "Advancing Deep Search Agents with Knowledge Graphs and Multi-Turn RL",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45250763",
         "created_at": "2025-09-15T15:22:49Z"
       },
       {
         "hn_id": "44980047",
         "title": "DuPO: Enabling Reliable LLM Self-Verification via Dual Preference Optimization",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44980047",
         "created_at": "2025-08-22T01:07:28Z"
       },
       {
         "hn_id": "44032309",
         "title": "How AI Generates Creativity from Inauthenticity",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44032309",
         "created_at": "2025-05-19T17:29:24Z"
       },
       {
         "hn_id": "41598858",
         "title": "An Imperative Language for Verified Exact Real-Number Computation",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41598858",
         "created_at": "2024-09-20T04:41:52Z"
       }
     ],
     "top_points": 324,
     "total_points": 364,
     "total_comments": 87
   }
 }