ai-research-survey

scan-v5.json (21278B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
     "authors": [
       "Xinyi Hou",
       "Yanjie Zhao",
       "Yue Liu",
       "Zhou Yang",
       "Kailong Wang",
       "Li Li",
       "Xiapu Luo",
       "David Lo",
       "John Grundy",
       "Haoyu Wang"
     ],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2308.10620",
     "doi": "XXXXXXX.XXXXXXX"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about selecting and analyzing 395 papers (2017–2024), answering four RQs, and publicly available artifacts are all verified in the body of the paper with detailed methodology sections and appendices.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive systematic review; no causal claims requiring experimental justification are made. Trend observations (e.g., decoder-only dominance in 2023) are presented as correlational observations, not causal findings.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Findings are explicitly scoped to the 395 papers in their corpus from January 2017 to January 2024, with clear acknowledgment in the threats-to-validity section that keyword incompleteness may cause omissions.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "When the authors attribute trends (e.g., 'This rapid transition can be linked to the generative capability of decoder-only LLMs'), they do not consider alternative explanations such as API availability, commercial incentives, or sampling bias from including 241 arXiv preprints.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures paper counts, venue distributions, and reported techniques; claims stay at this bibliometric level without conflating frequency of study with effectiveness or quality of the studied methods.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' is a dedicated section covering three distinct threats: paper search omission, study selection bias, and empirical knowledge bias.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named with concrete mitigations: keyword incompleteness addressed by QGS + snowballing, automated mislabeling addressed by retaining borderline papers for manual review, subjective bias addressed by inviting two external reviewers for secondary review.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Time frame (2017–2024) is explicitly justified by the Transformer inception date; the review explicitly excludes papers leveraging SE methods to enhance LLMs (as opposed to LLMs for SE) and lists exclusion criteria in Table 3.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section is present in the provided paper text; the paper lists institutional affiliations but makes no disclosure of grants, industry funding, or other financial support.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All ten authors have their institutional affiliations explicitly listed on the title page (HUST, Monash, SMU, Beihang, PolyU).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "Funding is not disclosed, so independence of funder from outcome cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears anywhere in the paper text.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "LLMs are defined and distinguished from general PLMs by parameter scale; architectures (encoder-only, encoder-decoder, decoder-only) are taxonomized and described in Section 3.1; SE tasks are organized into six SDLC phases.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The introduction states five specific contributions explicitly as bullet points, including being the first comprehensive SLR on 395 LLM4SE papers covering LLM taxonomy, datasets, optimization, evaluation, and 85 SE tasks.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 explicitly compares this work against eight prior surveys (Zhang et al., Zheng et al., Fan et al., Wang et al., etc.) on dimensions of model scope, SE scope, SLR status, time frame, and paper count, clearly situating the contribution.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "The Quasi-Gold Standard (QGS) approach is fully described with the complete SE and LLM keyword lists explicitly provided, enabling reproduction of the automated search.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 lists three inclusion criteria and nine exclusion criteria with specific, concrete conditions (e.g., 'short papers with fewer than 8 pages', 'non-English literature').",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly follows Kitchenham et al.'s SLR methodology (cited as [197, 198]), a well-established SE systematic review protocol, with planning, conducting, and reporting phases.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "Both keyword sets are fully listed in Section 2.2.1: SE task keywords (30+ terms) and LLM keywords (30+ terms) including wildcards, with explanation of why broad terms like 'ML' and 'DL' were included.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Seven databases are explicitly named with paper counts: IEEE Xplore (1,192), ACM Digital Library (10,445), ScienceDirect (62,290), Web of Science (42,166), Springer (85,671), arXiv (9,966), DBLP (4,035).",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Figure 1 shows a detailed flowchart with counts at every stage: 218,765 → 80,611 → 5,078 → 1,172 → 810 → 594 → 382 after quality assessment, then snowballing adds 13 for a final 395.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "The 2017 start date is justified by the publication of the Transformer architecture paper; arXiv inclusion is justified by the field's rapid evolution; top-venue selection is justified by targeting authoritative and comprehensive SE coverage.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": false,
           "justification": "The paper synthesizes findings descriptively by counting and categorizing, but does not systematically identify or discuss conflicting results across reviewed papers (e.g., papers reporting different conclusions about the same LLM on similar tasks).",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 defines ten Quality Assessment Criteria (QAC1–QAC10) with a three-point scoring rubric (0/1/2/3), and papers below a threshold score (80% of maximum) are excluded from the final corpus.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Publication bias—the tendency for positive results to be over-represented in published literature—is not discussed anywhere in the threats-to-validity section or elsewhere, despite being a standard concern in systematic reviews of empirical work.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": true,
           "justification": "The paper provides substantial quantitative synthesis through vote counting: frequency tables for LLMs per SE task, paper counts per venue and year, input form distributions, evaluation metric frequencies, and trend analysis across architectures over time.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Section 8's challenges and opportunities are each supported by specific citations to reviewed papers; for example, the recommendation to address industrial dataset scarcity is backed by the finding that only 6 of 395 studies used industrial datasets.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Decoder-only LLM architectures dominated LLM4SE research in 2023, comprising 70.7% of papers that year.",
       "evidence": "Figure 5 shows 432 decoder-only instances across 195 unique papers in 2023 vs. 85 encoder-decoder and 94 encoder-only instances.",
       "supported": "strong"
     },
     {
       "claim": "Code generation is the most studied SE task, appearing in 118 of 395 papers.",
       "evidence": "Table 10 lists code generation under software development with 118 papers, far exceeding the next task (program repair at 35).",
       "supported": "strong"
     },
     {
       "claim": "61% of selected papers (241 of 395) are arXiv preprints rather than peer-reviewed venue publications.",
       "evidence": "Figure 2(a) shows 241 arXiv papers vs. 154 peer-reviewed venue papers; the text explicitly notes this.",
       "supported": "strong"
     },
     {
       "claim": "Only 6 of the 395 studies used industrial datasets, indicating a gap between academic research and real-world deployment contexts.",
       "evidence": "Figure 6 shows industrial datasets used in only 6 papers; the text notes this 'suggests a potential misalignment between academic and industrial research.'",
       "supported": "strong"
     },
     {
       "claim": "Token-based input forms account for approximately 97.75% of LLM4SE studies that specify input format.",
       "evidence": "Table 8 shows 347 token-based papers out of 355 that specify input form (150 text tokens + 118 code tokens + 78 combined + rounding).",
       "supported": "strong"
     },
     {
       "claim": "LLM4SE research grew explosively in 2023, with 273 relevant papers published in that year alone.",
       "evidence": "Figure 2(b) shows paper counts: 7 (2020), 13 (2021), 56 (2022), 273 (2023), 46 (Jan 2024 only).",
       "supported": "strong"
     },
     {
       "claim": "Software development tasks (primarily code generation) account for 56.65% of LLM4SE applications, while software management tasks account for only 0.69%.",
       "evidence": "Figure 10(a) shows these percentage breakdowns across six SDLC phases.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "This SLR of 395 LLM4SE papers (2017–2024) finds that decoder-only architectures (especially GPT and ChatGPT variants) have come to dominate SE research since 2022, with code generation being the single most studied task (118 papers). The field is characterized by heavy reliance on open-source datasets (62.83% of studies), near-total dependence on token-based inputs (97.75%), and strikingly little use of industrial datasets (only 6 studies), suggesting a significant gap between academic benchmarks and real-world software practice. Research volume exploded in 2023 with 273 papers, driven largely by arXiv preprints (61% of the corpus) that bypass peer review—a structural feature of the corpus not critically examined by the authors.",
   "red_flags": [
     {
       "flag": "Majority arXiv preprints",
       "detail": "61% of included papers (241/395) are unreviewed arXiv preprints. While the authors apply a quality rubric, including this many preprints introduces unknown reliability variation and likely inflates optimistic performance claims common in preprints."
     },
     {
       "flag": "Publication bias unaddressed",
       "detail": "The survey never discusses publication bias—the well-known tendency for positive LLM benchmark results to be published while failures or negative comparisons are not. Given the fast-moving commercial stakes of LLM4SE, this omission is significant."
     },
     {
       "flag": "No conflict synthesis",
       "detail": "Conflicting results across reviewed papers are not identified or reconciled. The survey reports what papers claim about LLM capabilities without noting when studies reach opposite conclusions about the same model or task."
     },
     {
       "flag": "Funding not disclosed",
       "detail": "No funding acknowledgment appears in the paper, making it impossible to assess potential industry influence on framing or paper inclusion decisions."
     },
     {
       "flag": "Quality rubric assesses reporting, not methodology",
       "detail": "The 10-criterion QAC in Table 4 assesses whether papers describe their setups clearly, not whether evaluations are methodologically sound (e.g., appropriate baselines, proper test sets, contamination controls). Low-quality evaluations that are well-described pass the filter."
     }
   ],
   "cited_papers": [
     {
       "title": "Guidelines for performing systematic literature reviews in software engineering",
       "relevance": "The methodological foundation for the entire review; the authors explicitly follow Kitchenham et al.'s three-phase SLR process."
     },
     {
       "title": "Machine/deep learning for software engineering: A systematic literature review",
       "relevance": "Primary precursor survey (Wang et al. 2022) covering ML/DL for SE with 1,209 papers; this paper positions itself as the LLM-specific successor."
     },
     {
       "title": "A survey of deep learning for software engineering",
       "relevance": "Yang et al. 2022 DL4SE survey; authors consulted it when formulating RQs and use its task taxonomy."
     },
     {
       "title": "Software Testing with Large Language Model: Survey, Landscape, and Vision",
       "relevance": "Directly related narrow SLR (Wang et al. 2023) on LLMs for testing with 52 papers; compared in Table 1."
     },
     {
       "title": "A Survey of Learning-based Automated Program Repair",
       "relevance": "Related narrow SLR (Zhang et al. 2023) on APR; used for comparison and as source of QGS papers."
     },
     {
       "title": "Evaluating large language models trained on code",
       "relevance": "Chen et al. 2021 (Codex/HumanEval); one of the most-cited primary studies (referenced 62 times in the corpus) and the source of the HumanEval benchmark."
     },
     {
       "title": "A survey of large language models",
       "relevance": "Zhao et al. 2023 background survey on LLMs generally; used to frame the LLM definition and scope."
     },
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "relevance": "Fan et al. 2023 concurrent survey compared in Table 1; does not follow systematic review protocol, which this paper addresses."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly helps practitioners and researchers navigate which LLMs and techniques to use for which SE tasks, with 85 tasks catalogued and trend data."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms expected trends (GPT dominance, code generation focus) rather than challenging conventional wisdom; the industrial dataset scarcity finding is the most noteworthy surprise."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Touches on security concerns (vulnerability detection, jailbreaks, PII leakage) but only in passing as future research directions, not as primary findings."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "A standard academic survey with no controversy, competing claims, or dramatic findings; even the criticism of prior surveys is mild."
     },
     "demo_ability": {
       "score": 1,
       "justification": "A public GitHub repository with replication artifacts is provided, though the survey itself is not a demonstrable tool."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Extensively covers GPT-4, ChatGPT, Codex, GitHub Copilot, and other high-profile models; authors are from well-known SE research groups (HUST, SMU, Monash)."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "36012320",
         "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
         "points": 34,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=36012320",
         "created_at": "2023-05-20T14:43:30Z"
       },
       {
         "hn_id": "41391420",
         "title": "Kan 2.0: Kolmogorov-Arnold Networks Meet Science",
         "points": 8,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41391420",
         "created_at": "2024-08-29T14:54:10Z"
       },
       {
         "hn_id": "36002796",
         "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
         "points": 6,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36002796",
         "created_at": "2023-05-19T15:02:59Z"
       },
       {
         "hn_id": "36585056",
         "title": "Natural Selection Favors AIs over Humans",
         "points": 4,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=36585056",
         "created_at": "2023-07-04T11:08:02Z"
       },
       {
         "hn_id": "32632312",
         "title": "Exploring the Role of the Cybercrime Underground in the Russia-Ukraine Conflict",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=32632312",
         "created_at": "2022-08-28T21:36:55Z"
       },
       {
         "hn_id": "37866902",
         "title": "Getting Bored of Cyberwar",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=37866902",
         "created_at": "2023-10-13T05:03:06Z"
       },
       {
         "hn_id": "36008023",
         "title": "Tree-of-Thought (ToT), complex and general problem solving with LLMs",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=36008023",
         "created_at": "2023-05-19T23:38:37Z"
       },
       {
         "hn_id": "37990040",
         "title": "Exploring the Space of Key-Value-Query Models with Intention",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37990040",
         "created_at": "2023-10-23T18:58:38Z"
       },
       {
         "hn_id": "36381655",
         "title": "Natural Selection Favors AIs over Humans",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36381655",
         "created_at": "2023-06-18T16:23:55Z"
       },
       {
         "hn_id": "36149566",
         "title": "Tree of Thoughts: Official vs. Wrong Implementation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36149566",
         "created_at": "2023-06-01T11:00:31Z"
       }
     ],
     "top_points": 34,
     "total_points": 70,
     "total_comments": 8
   }
 }