ai-research-survey

scan.json (20902B)

---

 {
   "paper": {
     "title": "Large Language Models in Software Documentation and Modeling: A Literature Review and Findings",
     "authors": ["Lukás Radoský", "Ivan Polasek"],
     "year": 2026,
     "venue": "SAMI 2026 (IEEE 24th World Symposium on Applied Machine Intelligence and Informatics)",
     "arxiv_id": "2602.04938"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "Literature review of 57 papers from TSE, TOSEM, EMSE, and ICSE (2024-2025) on LLMs for software documentation and modeling. Papers span 11 task categories, with code summarization (12 papers) and commit message generation (7 papers) most studied. Zero-shot prompting dominates; advanced techniques like chain-of-thought and multi-agent systems receive little attention. The authors conclude LLMs have led to evolutionary improvements rather than revolutionary changes in SE documentation and modeling workflows.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code, analysis scripts, or repository link provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The list of 57 analyzed papers is not released as structured data. Only references are provided in standard bibliography format."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tool specifications provided. A survey could release analysis tools with environment details, but none are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions. Section 3 describes the methodology at a high level but does not provide step-by-step reproducible instructions (e.g., exact search queries, screening forms)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments or statistical analysis of its own."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper makes no comparative statistical claims requiring tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted; survey summarizes existing literature descriptively."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments; the 'sample' is the set of papers found via literature search, not an experimental sample."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs to report variance across."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "Section 2 lists related SLRs but does not formally compare this survey's coverage, methodology, or findings against them. The comparison is descriptive (what topics others cover) without structured comparison of results."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No formal baseline comparison is conducted against prior surveys, so contemporariness cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey has no components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey does not evaluate a system using metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to a literature review's claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test set applicable to a survey paper."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows paper counts per task category (11 categories). Sections 4.1-4.11 provide per-category analysis of approaches, datasets, and prompting techniques."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss failure cases or where the reviewed approaches break down. It mostly summarizes what each paper proposes without critical analysis of limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 notes that 'the long-awaited revolution stemming from the introduction of LLMs might be lagging behind its hype' and that LLMs lead to 'evolution rather than revolution.' Section 4.4 notes that zero-shot LLMs were outperformed by BERT-based models for emotion detection [51]."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a literature review analyzing articles from four major venues, organized per tasks, with an overview of prompt techniques, metrics, and datasets. Sections 3-5 deliver on each of these claims."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. All findings are descriptive summaries of the literature."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 explicitly bounds scope to four venues (TSE, TOSEM, EMSE, ICSE), publications from 2024-2025, and tasks related to software documentation and modeling. The title could be read more broadly but the methodology section clearly states the boundaries."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents observations (e.g., zero-shot dominance, evolution over revolution) without discussing alternative explanations. For instance, the dominance of zero-shot prompting could be due to API costs, simplicity bias, or publication norms — none of which are explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims LLMs lead to 'evolution rather than revolution' based on counting and categorizing 57 papers, but does not discuss what would constitute 'revolution' vs 'evolution' or whether paper counts are an adequate proxy for assessing the transformative impact of LLMs on SE."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey does not use any models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey does not use models with hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3 describes three phases (venue identification, keyword search, manual reading) with filtering criteria (keywords: LLM, GPT, BERT, etc.) but provides no paper counts at each filtering stage. The reader cannot determine how many papers were screened at each phase to arrive at 57."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. The conclusion (Section 6) briefly notes future directions but does not discuss limitations of the review itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. Key unaddressed threats include: venue selection bias (only 4 venues), temporal limitation (2024-2025 only), keyword-based filtering potentially missing relevant papers, and manual screening subjectivity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 explicitly states scope: four venues (TSE, TOSEM, EMSE, ICSE), 2024-2025 publications, and tasks related to software documentation and modeling (broadly defined to include code summarization)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No structured dataset of the 57 analyzed papers with their categorizations, extracted attributes, or coding scheme is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the three-phase collection process: venue identification, keyword-based search (LLM, language model, GPT, BERT), and manual abstract/full-text review for relevance."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is published papers from standard SE venues."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While Section 3 describes three filtering phases with criteria, no counts are provided at each stage (e.g., total papers in venues → after keyword filtering → after manual review → 57 final). The pipeline lacks quantitative traceability."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section states: 'This research and paper was 100% funded by the EU NextGenerationEU through the Recovery and Resilience Plan for Slovakia under the project \"InnovAIte Slovakia\"' with project number."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are clearly listed as from Department of Applied Informatics, Faculty of Mathematics, Physics and Informatics, Comenius University Bratislava, with ORCID IDs."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "EU NextGenerationEU funding through Slovakia's Recovery and Resilience Plan. The funder has no commercial stake in the survey's findings about LLM usage in SE."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no model evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no model evaluation on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no method with inference cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram, no registered protocol, no reproducible search queries. Section 3 describes a 3-phase process (venue selection, keyword search, manual reading) but lacks the rigor of a structured review protocol — no search strings, no database queries, no screening counts."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "No quality scoring, risk-of-bias assessment, or structured evaluation of included studies. All 57 papers are treated equally regardless of their methodological quality."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The paper notes that 'most papers report improvement over state-of-the-art approaches, often significant' (Section 6) but does not consider whether this reflects publication bias toward positive results."
       }
     }
   },
   "claims": [
     {
       "claim": "Code summarization is the most studied documentation/modeling task for LLMs, with 12 papers in the analyzed corpus.",
       "evidence": "Figure 1 shows paper counts per category. Code summarization leads with 12 papers, followed by commit message generation and code commenting/reviewing/logging at 7 each.",
       "supported": "strong"
     },
     {
       "claim": "Zero-shot prompting is the dominant approach in LLM4SE documentation and modeling research, with few-shot and chain-of-thought receiving less attention.",
       "evidence": "Sections 4.1-4.11 repeatedly note zero-shot as the default choice across task categories. Section 6 states: 'The popular approaches involve promptless models and zero-shot prompting; less attention is given to more advanced techniques such as few-shot prompting and chain-of-thought.'",
       "supported": "moderate"
     },
     {
       "claim": "LLMs have led to evolution rather than revolution in SE documentation and modeling tasks.",
       "evidence": "Section 6 states: 'the LLMs are being used to improve the speed and quality of existing workflow and task structure in software engineering instead of redefining them. At this point in time, their incorporation seems to lead to evolution rather than revolution.'",
       "supported": "weak"
     },
     {
       "claim": "LLM-driven multi-agent systems have received little attention in the documentation and modeling domain.",
       "evidence": "Section 6 notes: 'the emerging area of LLM-driven multi-agent systems has been given little attention, several papers mention them as a possible future research direction.' Only one paper [84] uses a MAS approach among the 57 analyzed.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of sources",
       "detail": "The survey treats all 57 papers equally without assessing their methodological quality, risk of bias, or strength of evidence. This risks laundering weak or poorly designed studies alongside rigorous ones."
     },
     {
       "flag": "No PRISMA or structured protocol",
       "detail": "The paper selection is described informally in Section 3 with no PRISMA flow diagram, no registered protocol, and no paper counts at filtering stages. The screening process is not reproducible."
     },
     {
       "flag": "Very shallow synthesis",
       "detail": "The paper reads primarily as a categorized bibliography — listing what each paper does — rather than providing critical synthesis, cross-paper comparison, or identification of systematic gaps. Most section content is descriptive summation of individual papers."
     },
     {
       "flag": "Sweeping conclusion unsupported by evidence",
       "detail": "The 'evolution rather than revolution' conclusion in Section 6 is presented as a finding but is really an editorial opinion. No framework is provided for what would constitute 'revolution' vs 'evolution,' making the claim unfalsifiable."
     },
     {
       "flag": "No limitations section",
       "detail": "A survey of this type should discuss venue selection bias (only 4 venues), temporal limits, keyword-based filtering gaps, and single-reviewer screening. None of these threats to validity are addressed."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["Xinyi Hou"],
       "year": 2024,
       "relevance": "Comprehensive SLR on LLM4SE across all SDLC phases — a key prior survey to position against."
     },
     {
       "title": "A survey on large language models for code generation",
       "authors": ["Juyong Jiang", "Fan Wang", "Jiasi Shen", "Sungju Kim", "Sunghun Kim"],
       "year": 2025,
       "relevance": "Survey focused on LLM code generation, directly relevant to the code-related tasks reviewed."
     },
     {
       "title": "A survey on large language models for software engineering",
       "authors": ["Quanjun Zhang"],
       "year": 2024,
       "relevance": "Another broad LLM4SE survey providing complementary coverage."
     },
     {
       "title": "The impact of llm-assistants on software developer productivity: A systematic literature review",
       "authors": ["Amr Mohamed", "Maram Assi", "Mariam Guizani"],
       "year": 2025,
       "relevance": "SLR on LLM impact on developer productivity — relevant to the productivity claims in the AI programming literature."
     },
     {
       "title": "LLM-based multi-agent systems for software engineering: Literature review, vision, and the road ahead",
       "authors": ["Junda He", "Christoph Treude", "David Lo"],
       "year": 2025,
       "relevance": "Survey on LLM multi-agent systems for SE — directly relevant to the agentic workflow dimension."
     },
     {
       "title": "Systematic literature review of prompt engineering patterns in software engineering",
       "authors": ["Yuya Sasaki", "Hironori Washizaki", "Jialong Li", "Dominik Sander", "Nobukazu Yoshioka", "Yoshiaki Fukazawa"],
       "year": 2024,
       "relevance": "SLR on prompt engineering patterns in SE — directly relevant to the prompting technique analysis."
     },
     {
       "title": "How much space do metrics have in GenAI assisted software development?",
       "authors": ["Samarth Sikand"],
       "year": 2024,
       "relevance": "Study on metrics in GenAI-assisted SE, relevant to evaluating how the field measures LLM impact."
     },
     {
       "title": "Software development life cycle perspective: A survey of benchmarks for code large language models and agents",
       "authors": ["Kaixing Wang"],
       "year": 2025,
       "relevance": "Survey of benchmarks for code LLMs and agents across the SDLC."
     },
     {
       "title": "Novice developers' perspectives on adopting LLMs for software development: A systematic literature review",
       "authors": ["Samuel Ferino", "Rashina Hoda", "John Grundy", "Christoph Treude"],
       "year": 2025,
       "relevance": "SLR on novice developer adoption of LLMs — relevant to the human factors dimension of LLM4SE."
     },
     {
       "title": "From LLMs to LLM-based agents for software engineering: A survey of current, challenges and future",
       "authors": ["Haolin Jin"],
       "year": 2025,
       "relevance": "Survey comparing standalone LLMs vs LLM-based agents for SE tasks."
     }
   ]
 }