ai-research-survey

scan.json (21609B)

---

 {
   "paper": {
     "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
     "authors": [
       "Xinyi Hou",
       "Yanjie Zhao",
       "Yue Liu",
       "Zhou Yang",
       "Kailong Wang",
       "Li Li",
       "Xiapu Luo",
       "David Lo",
       "John Grundy",
       "Haoyu Wang"
     ],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2308.10620"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository: https://github.com/xinyi-hou/LLM4SE_SLR, mentioned in the abstract and Section 3.1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at the GitHub repository includes the list of LLMs and their parameter sizes, and the paper list. Referenced in Sections 3.1 and 7."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. This is a survey, but the analysis scripts (if any) have no documented environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided for replicating the search, filtering, or analysis procedures beyond the methodology description in Section 2."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not run experiments or report quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical comparisons are made; the paper reports descriptive counts and distributions of papers across categories."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental effects are measured; this is a survey paper reporting distributions and trends."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experimental sample sizes; the paper aims for comprehensive coverage of relevant literature via systematic search."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are run; descriptive statistics of paper counts do not require variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares this SLR against 8 prior surveys, positioning it relative to existing reviews by scope, time frame, number of papers, and whether they follow an SLR process."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared surveys in Table 1 are from 2022-2023, which are contemporary to this 2024 work."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No system evaluation is performed; this is a literature review."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate; this is a survey."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring train/test splits; this is a survey."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive breakdowns: by SE activity (Fig. 10), by LLM architecture (Fig. 5), by data type (Table 7), by evaluation metric (Table 9), and by SE task (Table 10)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses challenges and limitations of LLM4SE including ambiguity in code generation, generalizability issues, evaluation challenges, and interpretability/trustworthiness concerns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports gaps: only 6 studies used industrial datasets (Section 4.1), software management has minimal coverage (0.69%, Section 6.1), and discusses limitations like models failing after semantic-preserving transformations (Section 8.1.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims analysis of 395 papers across four RQs, which is substantiated throughout Sections 3-6 with detailed breakdowns and tables."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims; it describes the landscape and trends of LLM4SE research without claiming causal relationships."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper states its scope as papers from January 2017 to January 2024, specifies the search databases used, and discusses limitations of the search process in Section 7 (paper search omission, study selection bias)."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/taxonomy paper presenting no empirical results that would require alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no measurements; it reports counts and categorizations of existing literature."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used in the methodology; this is a literature survey."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used; this is a literature survey."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No model training or inference is performed; this is a literature survey."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; this is a literature survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 and Figure 1 detail the full paper selection pipeline with counts at each stage: 218,765 initial papers → 80,611 after page filter → 5,078 after keyword screening → 1,172 after venue filtering → 810 after dedup → 382 after quality assessment → 395 final. Inclusion/exclusion criteria are stated in Table 3, and quality assessment criteria in Table 4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides a dedicated discussion of three types of threats: paper search omission, study selection bias, and empirical knowledge bias."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: incomplete keyword summarization may miss papers, BibTeX record ambiguity causing mislabeling, subjective judgments in quality assessment. They describe specific mitigation steps like having two experienced reviewers conduct secondary review."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly bounds scope to LLMs applied to SE tasks (not SE for LLMs), papers from 2017-2024, papers with 8+ pages, and excludes workshops, theses, and tool demos (Table 3). The scope exclusion of SE4LLM is explicitly noted in Section 8.3."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at https://github.com/xinyi-hou/LLM4SE_SLR contains the paper list and extracted data, referenced in Sections 3.1 and 7."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the search strategy in detail: manual search across 6 venues, automated search across 7 databases with specific search strings, and snowballing. The complete keyword sets are provided."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; this is a literature survey. Data sources are standard academic databases described in Section 2.2."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 documents the full pipeline with counts at each stage. Section 2.3 describes inclusion/exclusion criteria, quality assessment scoring, and the snowballing process. Specific thresholds are stated (e.g., QAC score ≥ 16.8/21 for published papers)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section is visible in the paper text provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Huazhong University of Science and Technology, Monash University, Singapore Management University, Beihang University, Hong Kong Polytechnic University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no method that incurs inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The paper follows Kitchenham et al.'s SLR methodology (cited in Section 2). It uses the Quasi-Gold Standard (QGS) approach with a detailed flow diagram (Figure 1), systematic search strings, and structured stages for identification, selection, and quality assessment."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 presents 10 Quality Assessment Criteria (QAC1-QAC10) scored on 0-3 scales. Papers needed ≥80% of maximum score to be included. Section 2.3.2 describes the full quality assessment process."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss publication bias. It acknowledges that 241/395 papers are from arXiv (not peer-reviewed) but does not discuss whether the included literature skews toward positive results or whether negative-result papers are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "395 relevant papers on LLM4SE were identified from January 2017 to January 2024, with 273 published in 2023 alone, showing rapidly growing research interest.",
       "evidence": "Section 2.5 and Figure 2(b) show the temporal distribution: 7 (2020), 13 (2021), 56 (2022), 273 (2023), 46 (January 2024).",
       "supported": "strong"
     },
     {
       "claim": "Decoder-only LLMs dominate SE tasks, with 432 instances across 195 papers in 2023, constituting 70.7% of research.",
       "evidence": "Section 3.2 and Figure 5 show the trend analysis with specific counts per architecture type per year.",
       "supported": "strong"
     },
     {
       "claim": "Code generation is the most prevalent SE task addressed by LLMs, with 118 studies, followed by program repair with 35 studies.",
       "evidence": "Table 10 provides the complete distribution of SE tasks, and Section 6.4 discusses code generation in detail.",
       "supported": "strong"
     },
     {
       "claim": "Only 6 out of 395 studies used industrial datasets, suggesting misalignment between academic research and industrial contexts.",
       "evidence": "Section 4.1 and Figure 6 report the dataset source distribution: 235 open-source, 49 collected, 84 constructed, 6 industrial.",
       "supported": "strong"
     },
     {
       "claim": "Software development accounts for 56.65% of LLM4SE research, while software management represents only 0.69%.",
       "evidence": "Section 6.1 and Figure 10(a) show the distribution across six SE activities.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This SLR of 395 papers (2017-2024) maps the LLM4SE landscape across four dimensions: models, data, optimization techniques, and SE tasks. Decoder-only architectures dominate (70.7% in 2023), with code generation (118 papers) and program repair (35 papers) as the most-studied tasks. The review identifies significant gaps: only 6 studies use industrial datasets, software management and design are barely explored, and the field lacks comprehensive evaluation frameworks. The paper catalogues 85 distinct SE tasks, 70+ LLMs, 8 prompt engineering techniques, and 19+ evaluation metrics used across the surveyed literature.",
   "red_flags": [
     {
       "flag": "No quality assessment of surveyed papers' methodology",
       "detail": "While the paper assesses source quality for inclusion (QAC criteria in Table 4), the assessment focuses on paper completeness and clarity rather than methodological rigor. The survey does not evaluate whether the 395 included papers use appropriate experimental designs, report statistics correctly, or address contamination. This risks laundering weak results by treating all included papers as equally valid evidence."
     },
     {
       "flag": "61% of included papers are non-peer-reviewed arXiv preprints",
       "detail": "241 of 395 papers are arXiv preprints. While the authors acknowledge this and apply quality assessment, the high proportion means the survey's conclusions rest heavily on unvetted work. The quality assessment criteria (QAC) are relatively permissive — scoring ≥80% on clarity/motivation/contribution criteria does not ensure methodological soundness."
     },
     {
       "flag": "Publication bias not discussed",
       "detail": "The survey does not consider whether the LLM4SE literature it reviews is biased toward positive results. Papers showing LLMs fail at SE tasks may be underrepresented, skewing the survey's characterization of LLM effectiveness."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark, foundational for LLM code generation evaluation."
     },
     {
       "title": "A systematic evaluation of large language models of code",
       "authors": ["Frank F Xu", "Uri Alon", "Graham Neubig", "Vincent Josua Hellendoorn"],
       "year": 2022,
       "relevance": "Systematic evaluation of multiple LLMs (Codex, GPT-J, GPT-Neo, PolyCoder) on SE tasks including code completion."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Introduces EvalPlus for more rigorous evaluation of LLM-generated code, addressing test adequacy."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM capability on real-world software engineering tasks."
     },
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "Key study on conversational automated program repair with ChatGPT, demonstrating cost-effectiveness."
     },
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["Angela Fan", "Beliz Gokkaya", "Mark Harman"],
       "year": 2023,
       "arxiv_id": "2310.03533",
       "relevance": "Competing survey on LLMs for SE, useful for comparison of survey scope and findings."
     },
     {
       "title": "Software Testing with Large Language Model: Survey, Landscape, and Vision",
       "authors": ["Junjie Wang"],
       "year": 2023,
       "arxiv_id": "2307.07221",
       "relevance": "Focused survey on LLMs for software testing, complementary to this broader SLR."
     },
     {
       "title": "A survey of large language models for code: Evolution, benchmarking, and future trends",
       "authors": ["Zibin Zheng"],
       "year": 2023,
       "arxiv_id": "2311.10372",
       "relevance": "Survey specifically on code LLMs with benchmarking focus, overlapping scope."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li"],
       "year": 2022,
       "relevance": "Demonstrates LLM capability at competition-level programming, significant for code generation evaluation."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-Trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Comprehensive study of LLMs for automated program repair across multiple benchmarks."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian"],
       "year": 2023,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent LLM system for software development, relevant to agentic AI workflows."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework for collaborative software development using LLMs."
     }
   ]
 }