ai-research-survey

scan.json (22408B)

---

 {
   "paper": {
     "title": "The Impact of LLM-Assistants on Software Developer Productivity: A Systematic Literature Review",
     "authors": ["Amr Mohamed", "Maram Assi", "Mariam Guizani"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.03156",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All artifacts associated with this study are publicly available at https://zenodo.org/records/15788502' and references [20] is the replication package."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo replication package [20] contains 'all study data, selection decisions, and exclusion rationales' per the contributions section."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or software versions are mentioned. The study is a manual SLR, but the Zenodo package does not describe any tool environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While data is released, no step-by-step reproduction instructions are described. The paper describes methodology but does not provide a guide for reproducing the analysis from the replication package."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that synthesizes qualitative findings via thematic analysis; it does not run experiments or produce quantitative results requiring CIs."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical comparisons are made; the paper counts and categorizes studies but does not perform statistical tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; findings are qualitative synthesis of primary studies."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The final set is 37 studies but the paper does not justify whether this is sufficient for the claims made or discuss saturation."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs; the SLR reports counts and thematic categories, not repeated measurements."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare its findings against prior SLRs or reviews on similar topics. It claims to be 'the first systematic literature review focused on the impact of LLM-assistants on software developer productivity' but does not position against related reviews."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baselines are included, so contemporariness is not applicable."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "An SLR has no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study; evaluation metrics are not applicable."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not relevant; this is a survey paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test sets in an SLR."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive breakdowns: by research strategy (Table 3), by procedure (Table 4), by SPACE dimension (Table 8, Figure 8-9), by benefit/risk theme (Figure 7, Tables 6-7), and by venue (Table 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses gaps and risks extensively in RQ2 (Section 6.2), including cases where LLM-assistants fail to improve or actually degrade productivity, code quality, and collaboration."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Mixed and negative findings are reported throughout: e.g., no significant difference in task completion time (PS2), increased frustration (PS10), negative correlation between throughput and quality (PS30, r=-0.45)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about benefits (minimized code search, accelerated development, automation), risks (cognitive offloading, reduced collaboration, inconsistent quality), and SPACE coverage (92% multi-dimensional, 14% beyond three) are all supported by data in Sections 4-7."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper synthesizes findings from primary studies and does not make its own causal claims. It reports what other studies found without asserting causation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its scope to 37 peer-reviewed studies from 2014-2024, explicitly states inclusion/exclusion criteria, and the threats to validity section acknowledges potential omissions from the selection criteria."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 (Threats to Validity) discusses study selection bias, SLR bias and repeatability, and classification rigor. The paper also notes throughout that inconsistent findings across studies may stem from 'diverse operationalizations of cognitive load, differences in participants' expertise, task design, and the capabilities of LLM-assistants.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses the proxy-outcome gap: Section 2 discusses how traditional metrics (LOC, velocity) fail to capture broader productivity, and adopts the SPACE framework precisely because 'a single metric cannot meaningfully capture productivity.' The paper frames its analysis around this multi-dimensional understanding."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used in this SLR's methodology."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this SLR."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No AI models or hyperparameters are involved in the SLR methodology."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the full filtering pipeline: 8,540 initial records → 331 duplicates removed → 8,209 screened → 8,005 excluded by title/abstract → 204 full-text → 172 excluded (with per-criterion counts: EC1=15, EC2=124, EC3=18, EC4=10) → 32 + 5 snowballing = 37. Inclusion/exclusion criteria are explicitly stated in Section 3.1.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Threats to Validity' provides a dedicated, substantive discussion of study selection bias, SLR bias and repeatability, and classification rigor."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific: initial search strings including 'performance' and 'efficiency' yielded false positives focused on technical benchmarks; mapping findings to SPACE required interpretive decisions since SPACE was not designed for human-LLM collaboration; screening was led by one author with validation by others."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope: peer-reviewed studies only (excluding grey literature, theses, workshops), English only, 2014-2024, focused on LLM-assistants' impact on developer productivity. Exclusion criteria EC1-EC4 are clearly enumerated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package on Zenodo [20] contains 'all study data, selection decisions, and exclusion rationales' enabling independent verification of the selection process."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full SLR methodology: four databases searched (ACM, IEEE Xplore, ScienceDirect, Web of Science), exact search strings provided in Table 1, search date (December 31, 2024), and the PRISMA flow chart in Figure 2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this SLR. The paper reviews other studies; data source is database searches of published literature."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is fully documented with counts at each stage (Figure 2 PRISMA diagram): 8,540 → 8,209 after dedup → 204 after title/abstract screening → 32 after full-text → 37 after snowballing. Exclusion counts per criterion are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: two at Queen's University (Canada) and one at Université du Québec à Montréal (Canada). No conflict with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an SLR that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation; contamination is not applicable to this SLR."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation in this SLR."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This SLR has no human participants. It reviews published studies."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this SLR."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this SLR."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper has inclusion/exclusion criteria for studies, which is covered under data_preprocessing_documented."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions in this SLR."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; no inference costs are relevant."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; no computational budget is relevant."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The paper follows PRISMA (Figure 2 is a PRISMA flow chart citing [49] Page et al. 2020) and explicitly follows Kitchenham and Charters [43] guidelines for SLRs in software engineering. Search strings are reproducible (Table 1) across four databases."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not assess the methodological quality of its 37 primary studies. It classifies them by strategy, procedure, and instruments (Section 5), but does not apply a quality scoring rubric or risk-of-bias assessment. All 37 studies are treated equally regardless of their rigor."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss publication bias. There is no funnel plot, no discussion of whether published studies skew toward positive findings about LLM-assistants, and no acknowledgment that unpublished negative results may be missing."
       }
     }
   },
   "claims": [
     {
       "claim": "92% of studies adopt a multi-dimensional perspective on productivity by examining at least two SPACE dimensions, but only 14% extend beyond three dimensions.",
       "evidence": "Section 7 and Figure 9: 34 out of 37 studies examine 2+ dimensions; only 5 out of 37 examine more than 3.",
       "supported": "strong"
     },
     {
       "claim": "Satisfaction (78%), Performance (65%), and Efficiency (59%) are the most frequently investigated SPACE dimensions, while Activity (27%) and Communication (35%) are underexplored.",
       "evidence": "Table 8 and Figure 8 provide per-dimension counts across all 37 studies.",
       "supported": "strong"
     },
     {
       "claim": "The majority of studies are exploratory: 64% have formative objectives and laboratory experiments are the most common strategy (41%).",
       "evidence": "Section 5.1 (Table 3) and Section 5.2 classify all 33 empirical studies by strategy and objective.",
       "supported": "strong"
     },
     {
       "claim": "Code quality is the only theme reported as both a benefit and a risk of LLM-assistants.",
       "evidence": "Section 6.1.7 (improve code quality) and Section 6.2.1 (limit code quality) with references to multiple primary studies on each side (Figure 7).",
       "supported": "strong"
     },
     {
       "claim": "LLM-assistants' most frequently reported benefits are minimizing online code search, accelerating development, and automating trivial/repetitive tasks.",
       "evidence": "Figure 7 radar plot and Table 6 synthesize benefit frequency across primary studies.",
       "supported": "moderate"
     },
     {
       "claim": "Cognitive load findings are mixed: some studies report reduced mental effort, others neutral effects, and one reports increased frustration.",
       "evidence": "Section 5.3.3 reviews 6 studies using NASA-TLX with divergent results: [PS11, PS26, PS36] report improvements, [PS3, PS15] neutral, [PS10] increased frustration.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This SLR of 37 peer-reviewed studies (2014-2024) finds that LLM-assistants offer benefits including reduced code search, faster development, and task automation, but also introduce risks of over-reliance, reduced team collaboration, and inconsistent code quality effects. While 92% of studies examine multiple SPACE dimensions, only 14% extend beyond three, with Communication and Activity particularly underexplored. The research landscape is predominantly exploratory (64% formative) and relies heavily on laboratory experiments (41%) and self-reported methods (91% use surveys/interviews).",
   "red_flags": [
     {
       "flag": "No quality assessment of included studies",
       "detail": "The SLR synthesizes findings from 37 primary studies without assessing their methodological quality. Studies with 10 participants are weighted equally with those involving thousands. This risks laundering weak results alongside strong ones."
     },
     {
       "flag": "No publication bias discussion",
       "detail": "The paper excludes grey literature and non-peer-reviewed work but does not discuss whether the resulting corpus skews toward positive findings about LLM-assistants. Given that many primary studies involve industry tool evaluations, positive publication bias is a significant concern."
     },
     {
       "flag": "Narrow search scope may miss relevant work",
       "detail": "The authors acknowledge removing 'performance' and 'efficiency' from search terms due to false positives, but this risks missing studies that frame developer productivity through those lenses. The threats section notes this but the mitigation (snowballing) adds only 5 papers."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["Xinyi Hou"],
       "year": 2024,
       "relevance": "Comprehensive SLR on LLMs in software engineering, broader scope than this paper."
     },
     {
       "title": "The SPACE of developer productivity: There's more to it than you think",
       "authors": ["N Forsgren"],
       "year": 2021,
       "relevance": "Foundational framework for multi-dimensional developer productivity used as the analytical lens in this paper."
     },
     {
       "title": "Taking Flight with Copilot: Early insights and opportunities of AI-powered pair-programming tools",
       "authors": ["Christian Bird"],
       "year": 2022,
       "relevance": "Early analysis of GitHub Copilot's impact on developer workflows and productivity."
     },
     {
       "title": "AI-assisted Code Authoring at Scale: Fine-tuning, deploying, and mixed methods evaluation",
       "authors": ["Vijayaraghavan Murali"],
       "year": 2024,
       "relevance": "Meta's internal LLM code completion system evaluation with acceptance rate metrics and deployment analysis."
     },
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["Albert Ziegler"],
       "year": 2022,
       "relevance": "GitHub's statistical analysis of code completion interaction metrics and their relationship to self-reported productivity."
     },
     {
       "title": "Beyond code generation: An observational study of chatgpt usage in software engineering practice",
       "authors": ["Ranim Khojah"],
       "year": 2024,
       "relevance": "Field study of ChatGPT usage patterns in professional development, finding expert consultation as dominant use case."
     },
     {
       "title": "Rocks coding, not development: A human-centric, experimental evaluation of LLM-supported SE tasks",
       "authors": ["Wei Wang"],
       "year": 2024,
       "relevance": "Controlled experiment evaluating LLM-assisted coding showing time savings but no quality improvement."
     },
     {
       "title": "A large-scale survey on the usability of ai programming assistants: Successes and challenges",
       "authors": ["Jenny T Liang", "Chenyang Yang", "Brad A Myers"],
       "year": 2024,
       "relevance": "Large-scale survey of AI programming assistant usability covering benefits and limitations from developer perspective."
     },
     {
       "title": "Reading between the lines: Modeling user behavior and costs in AI-assisted programming",
       "authors": ["Hussein Mozannar"],
       "year": 2024,
       "relevance": "CHI study modeling developer interaction states with Copilot, finding 51.5% time in LLM interaction states."
     },
     {
       "title": "Significant productivity gains through programming with large language models",
       "authors": ["Thomas Weber"],
       "year": 2024,
       "relevance": "Controlled experiment comparing LLM-assisted programming across web search, code completion, and chat conditions using SPACE framework."
     },
     {
       "title": "Ironies of generative AI: understanding and mitigating productivity loss in Human-AI interaction",
       "authors": ["Auste Simkute"],
       "year": 2025,
       "relevance": "Examines automation ironies in generative AI contexts where productivity gains are offset by evaluation overhead."
     },
     {
       "title": "The role of firm AI capabilities in generative AI-pair coding",
       "authors": ["Jacques Bughin"],
       "year": 2024,
       "relevance": "Industry study of 70 large companies finding negative correlation (r=-0.45) between LLM-assisted throughput and code quality."
     }
   ]
 }