ai-research-survey

scan.json (24996B)

---

 {
   "paper": {
     "title": "Quo Vadis, Code Review? Exploring the Future of Code Review",
     "authors": [
       "Michael Dorner",
       "Andreas Bauer",
       "Darja Šmite",
       "Lukas Thode",
       "Daniel Méndez",
       "Ricardo Britto",
       "Stephan Lukasczyk",
       "Ehsan Zabardast",
       "Michael Kormann"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2508.06879",
     "doi": "10.48550/arXiv.2508.06879"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["qualitative", "observational"],
   "key_findings": "A survey of 100 professional developers from five companies finds that 77% expect to spend the same or more time on code review in five years, with broader artifact coverage anticipated. Almost all respondents expect LLMs to become active participants in code review, either as reviewers or code authors. The authors identify three long-term risks: erosion of human understanding, accountability, and trust in collaborative software engineering.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states: 'All anonymized data and analysis scripts are publicly available on GitHub: github.com/michaeldorner/quo-vadis-code-review.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states all anonymized data is publicly available on GitHub alongside the analysis scripts."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or software version requirements are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a GitHub repository is provided, the paper itself contains no step-by-step reproduction instructions, README description, or guidance on how to replicate the analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as simple percentages (47%, 30%, 23%) with no confidence intervals or error bars despite the small sample size of 100."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares its distribution of review hours against two external datasets (Microsoft 2013, Stack Overflow 2019) and notes the distribution is 'consistently lower,' but performs no statistical test to assess whether the difference is significant."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Comparisons are made only through visual inspection of cumulative distribution plots and raw percentages."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The target of ~25 developers per company is stated as a quota but not justified. No power analysis or rationale for why 100 total participants is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only a median of 'approximately three hours per week' is reported for review time. No standard deviation, interquartile range, or other spread measure is provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper contextualizes its findings against two external datasets: Bosu et al. (2017) surveying 416 Microsoft developers in 2013, and the 2019 Stack Overflow Developer Survey with 49,790 respondents (Section 4.1, Figure 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The reference datasets are from 2013 (Microsoft) and 2019 (Stack Overflow), both 6+ years old. The paper acknowledges 'these datasets are older' but does not justify why no more recent reference data was used."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study with no system or components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The survey examines multiple dimensions: weekly review hours (Section 4.1), expected future time investment, types of artifacts reviewed today vs. in five years (Section 4.2), and qualitative expectations about AI involvement (Section 4.3)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate. The study itself is a survey collecting human perspectives, not evaluating a system's outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey study — there is no train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides per-artifact breakdowns (production code, test code, configuration files, documentation, GUI-based test code) comparing today vs. five years from now. Table 1 provides per-company participant counts."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses opposing views: fundamental opposition to AI in code review ('As long as AI is involved, it is likely to be a more tedious process'), warnings about degraded code quality ('more generated code of a worsen quality'), and the risk of both author and reviewer being AI ('the AI doesn't catch the mistake because it generated it')."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 23% of respondents expect to spend less time on code review (contrary to the main trend), and Section 5 is entirely devoted to negative implications (erosion of understanding, accountability, trust)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: 'similar or greater effort' matches 77% expecting same/more time (Section 4.1), 'broader range of artifacts' matches Figure 2 (Section 4.2), and 'almost all expect LLMs to become active participants' is supported by the thematic analysis (Section 4.3)."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. All empirical findings are descriptive (what developers report and expect). The discussion section uses hedged language: 'could lead to,' 'might happen,' 'we fear,' 'may have long-term implications.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'we are cautious about extrapolating this insight to the entire software industry' (Section 4.1), 'Though not statistically representative' (Section 2.2), and 'all findings reflect the perspectives of individual developers; the companies serve only as contextual backgrounds' (Section 2.2)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 briefly mentions 'Possible explanations may include a decline in the perceived importance of code review, differences in domain-specific practices, or inaccuracies in self-reported estimates' for the lower review hours. However, there is no systematic discussion of confounds such as social desirability bias, self-selection bias, or the reliability of 5-year predictions."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures self-reported expectations about the future and uses them to discuss what code review will actually look like. The gap between developer predictions and actual future outcomes is not acknowledged — self-reported expectations are an unreliable proxy for actual future practice changes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used in this study. It is a survey of human developers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting of AI models is involved in this study."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models or experiments with hyperparameters are involved."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.5 describes the analysis pipeline: quantitative responses analyzed with descriptive statistics, open-ended responses examined using thematic analysis with inductive single-cycle descriptive coding by the first author, reviewed by the second author, with disagreements resolved collaboratively."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Caveats are scattered throughout (e.g., 'not statistically representative' in Section 2.2, 'cautious about extrapolating' in Section 4.1), but there is no substantive consolidated discussion of study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are systematically discussed. The scattered caveats are relatively generic ('not statistically representative,' 'varying participation per company due to voluntary participation'). Specific threats like social desirability bias, self-selection, or the reliability of 5-year predictions are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper hedges about generalizability, it does not explicitly state what the results do NOT show. The title ('Exploring the Future of Code Review') and discussion sections make broad claims about erosion risks without bounding these to the 5-company sample. No equivalent of 'what the evidence does not show.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states: 'All anonymized data and analysis scripts are publicly available on GitHub: github.com/michaeldorner/quo-vadis-code-review.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 describes data collection: company-specific online questionnaires shared through internal communication channels, conducted between December 2024 and November 2025, with voluntary and anonymous participation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes quota sampling: five companies were purposively selected, then ~25 developers recruited from each via internal channels, with recruitment closing after eight weeks or once the quota was reached."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document whether any responses were excluded, filtered, or cleaned. There is no accounting of incomplete responses, nor explanation of how the raw survey data was processed into the reported statistics. The jump from 'we collected responses' to final numbers is undocumented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgement section states: 'This work was supported by the KKS Foundation through the SERT Project (Research Profile Grant 2018/010) at Blekinge Institute of Technology.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Stephan Lukasczyk is at JetBrains Research, Michael Kormann is at SAP, and Ricardo Britto is at Ericsson. These are three of the five surveyed companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The KKS Foundation (Knowledge Foundation) is a Swedish research funding body with no apparent commercial interest in code review practices or tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Three authors are affiliated with surveyed companies (JetBrains, SAP, Ericsson), which could influence how questions were framed or results interpreted, but this potential conflict is not acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a human survey study that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a human survey study that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a human survey study that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, or any registry) anywhere in the paper."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval despite collecting data from 100 human participants across five companies."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper deliberately excluded demographic questions: 'We deliberately excluded demographic questions such as age, gender, or company affiliation to avoid any mapping between individual respondents and their employer.' No experience level, years of experience, or other participant characteristics are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "The only stated criterion is that participants be 'professional software developers' from the five selected companies. No formal inclusion/exclusion criteria, screening process, or minimum experience requirements are documented."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experimental study with treatment/control conditions. Randomization does not apply."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experimental study. Blinding does not apply."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No response rate is reported. The paper does not state how many developers were invited vs. how many responded, nor whether any responses were excluded. Only the final participant counts per company are given (Table 1)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study with no computational method or inference to cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study with no computational experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "A majority of practitioners (77%) expect to spend the same or more time on code review in the coming years.",
       "evidence": "Section 4.1 reports 47% expect more time, 30% expect the same, and 23% expect less, based on the 100-developer survey.",
       "supported": "moderate"
     },
     {
       "claim": "Practitioners currently spend a median of approximately three hours per week on code review, lower than older reference datasets showing a median of around four hours.",
       "evidence": "Section 4.1 and Figure 1 compare the distribution against Bosu et al. (2017) Microsoft data and the 2019 Stack Overflow Developer Survey. The comparison is visual (cumulative distribution plot) with no formal statistical test.",
       "supported": "moderate"
     },
     {
       "claim": "Practitioners expect to review a broader range of artifacts in the future, with notable growth in GUI-based test code.",
       "evidence": "Section 4.2 and Figure 2 show increases across all artifact categories (production code, test code, configuration files, documentation, GUI-based test code) and a decrease in 'none' responses.",
       "supported": "moderate"
     },
     {
       "claim": "Almost all practitioners expect LLMs to become active participants in code review.",
       "evidence": "Section 4.3 reports this finding from thematic analysis of open-ended responses, but does not provide an exact percentage or count of how many respondents mentioned LLMs.",
       "supported": "weak"
     },
     {
       "claim": "LLM involvement in code review risks eroding human understanding, accountability, and trust.",
       "evidence": "Section 5 develops three erosion narratives supported by respondent quotes and references to related work (Tufano et al. 2025, Alami et al. 2025, Taivalsaari et al. 2025). The paper explicitly states these are 'conceptual interpretations informed by practitioner feedback and prior work rather than direct empirical findings.'",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Small, non-representative sample",
       "detail": "N=100 developers from only 5 purposively selected companies, with uneven participation (9–31 per company). The paper acknowledges it is 'not statistically representative' yet draws broad conclusions about the future of code review generally."
     },
     {
       "flag": "No response rate reported",
       "detail": "The paper does not state how many developers were invited at each company, making it impossible to assess self-selection bias. Developers who are more engaged with or opinionated about code review may be overrepresented."
     },
     {
       "flag": "Author-company overlap not discussed as conflict",
       "detail": "Three authors are affiliated with three of the five surveyed companies (JetBrains, SAP, Ericsson). This could influence survey design, distribution, or interpretation, but is not acknowledged as a potential conflict of interest."
     },
     {
       "flag": "No IRB or ethics approval",
       "detail": "The study collected data from 100 human participants across five companies with no mention of ethics board approval, despite this being standard for human subjects research."
     },
     {
       "flag": "Deliberately excluded demographics",
       "detail": "Demographics were excluded for anonymity, but this means there is no way to assess whether the sample is representative of professional developers or skewed toward particular experience levels, roles, or backgrounds."
     },
     {
       "flag": "Speculative discussion beyond empirical evidence",
       "detail": "Section 5's erosion narratives (understanding, accountability, trust) go substantially beyond what the survey data shows. The paper acknowledges these are 'conceptual interpretations' but presents them prominently, including in the abstract, alongside the empirical findings."
     }
   ],
   "cited_papers": [
     {
       "title": "Tales from the Trenches: Expectations and Challenges from Practice for Code Review in the Generative AI Era",
       "authors": ["N. Davila", "J. Melegati", "I. Wiese"],
       "year": 2024,
       "doi": "10.1109/MS.2024.3428439",
       "relevance": "Gray literature review identifying practitioner interest in generative AI solutions for code review, directly relevant to AI-assisted software engineering practices."
     },
     {
       "title": "Deep Learning-based Code Reviews: A Paradigm Shift or a Double-Edged Sword?",
       "authors": ["R. Tufano", "A. Martin-Lopez", "A. Tayeb", "O. Dabic", "S. Haiduc", "G. Bavota"],
       "year": 2025,
       "doi": "10.1109/ICSE55347.2025.00060",
       "relevance": "Empirical study on how AI-assisted code reviews affect reviewer behavior, finding reviewers focus on annotated sections and miss issues elsewhere."
     },
     {
       "title": "Future of Software Development with Generative AI",
       "authors": ["J. Sauvola", "S. Tarkoma", "M. Klemettinen", "J. Riekki", "D. Doermann"],
       "year": 2024,
       "doi": "10.1007/s10515-024-00426-z",
       "relevance": "Discusses the broader future of software development with generative AI, including accountability concerns when delegating engineering tasks to AI."
     },
     {
       "title": "Human and Machine: How Software Engineers Perceive and Engage with AI-Assisted Code Reviews Compared to Their Peers",
       "authors": ["A. Alami", "N. Ernst"],
       "year": 2025,
       "relevance": "Interview study showing developers experience AI code review comments as lacking contextual grounding and requiring greater cognitive effort to evaluate."
     },
     {
       "title": "Accountability in Code Review: The Role of Intrinsic Drivers and the Impact of LLMs",
       "authors": ["A. Alami", "V. Jensen", "N. Ernst"],
       "year": 2025,
       "relevance": "Empirical study showing that LLM-assisted reviews reduce developers' perceived ownership of code quality and shift responsibility toward the tool."
     },
     {
       "title": "On the Future of Software Reuse in the Era of AI Native Software Engineering",
       "authors": ["A. Taivalsaari", "T. Mikkonen", "C. Pautasso"],
       "year": 2025,
       "relevance": "Argues AI-generated code represents a new form of generative reuse where developers rely on artifacts they cannot fully understand, risking long-term maintainability."
     },
     {
       "title": "Modern Code Reviews - A Survey of Literature and Practice",
       "authors": ["D. Badampudi", "M. Unterkalmsteiner", "R. Britto"],
       "year": 2023,
       "doi": "10.1145/3585004",
       "relevance": "Comprehensive survey of modern code review literature and practice, providing baseline understanding of code review as currently practiced."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The paper reports developer expectations about code review's future but provides no actionable tool, framework, or technique that practitioners can apply."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that developers expect to spend the same or more time on code review despite AI assistance mildly challenges the 'AI replaces everything' narrative."
     },
     "fear_safety": {
       "score": 1,
       "justification": "The erosion narratives (understanding, accountability, trust) raise concerns about AI in software engineering but at a conceptual rather than demonstrated-threat level."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The 'unsupervised software engineering' framing and erosion warnings create mild tension but the paper's tone is measured and academic."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Anonymized survey data and analysis scripts are released on GitHub, but there is no interactive demo or tool to try."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "The survey includes developers from SAP, Ericsson, and JetBrains — well-known brands in the developer tools space that may attract attention."
     }
   }
 }