ai-research-survey

scan.json (28424B)

---

 {
   "paper": {
     "title": "Human and Machine: How Software Engineers Perceive and Engage with AI-Assisted Code Reviews Compared to Their Peers",
     "authors": ["Adam Alami", "Neil A. Ernst"],
     "year": 2025,
     "venue": "IEEE/ACM International Conference on Connected Health: Applications, Systems and Engineering Technologies (CHASE)",
     "arxiv_id": "2501.02092",
     "doi": "10.1109/CHASE66643.2025.00016"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No analysis code or scripts are released. A shared documents package at Zenodo (doi:10.5281/zenodo.14000259) contains code snippets, reviews, questionnaires, and member checking responses, but no analysis code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "A shared documents package is released at Zenodo (Sect. III-E) containing code snippets and reviews submitted by participants, the interview pre-screening questionnaire, ChatGPT-generated reviews, member checking questionnaire and responses, and the interview guide. Interview transcripts are promised on acceptance but not yet available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No specification of analysis tools or their versions. The paper mentions using Otter.ai for transcription and Zoom for interviews but provides no version information or reproducibility details for the qualitative analysis environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the analysis. The methods section (Sect. III) describes the general process at a high level but does not provide specific instructions a researcher could follow to replicate the coding and analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study with no quantitative results requiring confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative study that makes no statistical comparisons. All findings are thematic, derived from qualitative coding."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative effect sizes are relevant to this qualitative interview study."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The authors monitored data saturation throughout the analysis (Sect. III-C): 'we managed to observe when our Pattern Codes reoccur strongly in the data, hence reaching saturation.' They documented the saturation process and cite standard qualitative methodological references for this approach."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative variance measures are relevant to this qualitative study."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study design is inherently comparative: it juxtaposes software engineers' engagement with human peer reviews against LLM-generated reviews of the same code, providing a built-in comparison condition."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison uses ChatGPT 4.0 (2024), which was contemporary at the time of the study (interviews conducted September 2024)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative study that does not use quantitative metrics. Findings are organized by thematic dimensions (cognitive, emotional, behavioral) rather than measured with metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire study centers on human evaluation: 20 software engineers evaluated and compared LLM-generated code reviews against peer-generated reviews, providing detailed qualitative assessments of both."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a qualitative interview study — there are no test sets or validation splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Findings are broken down across multiple dimensions: cognitive, emotional, and behavioral engagement (Sect. IV-B), feedback content and delivery (Sect. IV-A), reviewer context (Sect. IV-C), and human-AI collaboration preferences (Sect. IV-D). Table III documents pattern codes and first cycle codes."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where LLM-assisted review falls short: higher cognitive load (P14), lack of codebase context (P18, P20), trust issues (P20), and excessive verbosity (P5's signal-to-noise concern). These are substantive failure cases of the AI integration."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings about LLM-assisted review are reported: higher cognitive load ('ChatGPT review will take more time and effort to analyze,' P14), trust constraints ('I wouldn't trust ChatGPT,' P20), and lack of context ('AI might not be completely aware of what the context of the code you are writing,' P18)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims — multi-dimensional engagement (cognitive, emotional, behavioral), less emotional regulation needed with LLM, higher cognitive load with LLM feedback, similar sense-making process, trust and context constraints — are all supported by findings in Section IV with participant quotes."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses causal language: 'The introduction of LLM-assisted review impacts some of these attributes' (abstract), 'feedback delivery itself triggers negative emotional responses' (Sect. V-B). The within-subjects comparison provides some support, but participants were not blinded to review source (they knew which was ChatGPT), order was not counterbalanced (peer reviews always first), and the study design cannot control for confounds like novelty or expectation effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to study 'How Software Engineers Perceive and Engage with AI-Assisted Code Reviews' broadly, but the study involves only 20 Prolific-recruited participants in an artificial anonymous setting. While the paper uses hedging language ('in our sample') within the text, the title and implications (Sect. V) generalize broadly to 'SE' without adequate bounding."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses specific alternative explanations: anonymity may have altered review behavior ('developers may align the tone of their feedback according to status, relationships, and team culture'), the artificial setting may not capture real-world complexity, and participants may have modified their approach given the research context."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines its construct: 'Engagement, in the context of our study, is the ways and the extent to which software engineers actively interact with, respond to, and incorporate feedback from the review process' (Sect. I). Self-reported perceptions are clearly framed as the measurement approach for this construct, and the paper does not overstate what interview data can show."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'ChatGPT 4.0' throughout without specifying an API version, snapshot date, or model identifier (e.g., gpt-4-0613). 'ChatGPT 4.0' is ambiguous — it could refer to GPT-4 or GPT-4o, and model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The exact prompt is provided in Sect. III-B: 'You are an expert of [the programming language]. Provide a thorough review of the attached code.' The programming language fill values are documented in Table I for each participant. The full generated reviews are also shared in the Zenodo package."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, max tokens, or other API settings used when generating ChatGPT reviews."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding was used. ChatGPT was prompted directly with a single prompt per code submission."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from recruitment to analysis is documented: 500 initial participants → 353 after pre-screening → 76 after pre-selection → 20 interviewed (Sect. III-A). The analysis pipeline (First Cycle coding → Pattern Codes → reliability check → member checking) is also described in detail (Sect. III-C)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Limitations and Trade-offs' is a dedicated section discussing anonymity constraints, artificial setting, and participant behavior modification."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: Prolific's anonymity requirement differs from real-world code review where 'developers are colleagues and interact directly on a daily basis'; the controlled environment 'may not capture the inherent complexity of a real-world setting'; participants may have 'modified their approach to the review compared to a professional setting.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses threats but does not explicitly state what the results do NOT show or what populations/settings are excluded. There is no statement bounding the scope of the findings (e.g., not applicable to in-house teams, open-source communities, specific industries, etc.)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Interview transcripts (the core qualitative data) are not available — the paper states 'Interview transcripts will be published on acceptance' (Sect. III-E footnote 3). Supporting materials are available at Zenodo but the raw analytical data cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail (Sect. III-B): semi-structured interviews via Zoom, 40-60 minutes each, 17h12min total audio, 271 pages verbatim transcription using Otter.ai, conducted in first and second weeks of September 2024."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Recruitment is thoroughly documented (Sect. III-A): Prolific platform used, iterative pre-screening with programming tasks and critical-thinking questions following Alami et al.'s guidelines, AI-content detection applied, compensation specified (£0.50 for surveys, £60.00 for interviews). Potential Prolific platform bias is acknowledged."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from recruitment (500 cap) → pre-screening (353 qualified) → pre-selection (76 agreed) → interviews (20 completed) → transcription → First Cycle coding → Pattern Codes → reliability check → member checking is fully documented with counts at each stage (Sects. III-A through III-C)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgment: 'This study was funded by the department of computer science at Aalborg University; research funding for tenure-track assistant professors.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Adam Alami at University of Southern Denmark, Neil Ernst at University of Victoria. Neither is affiliated with OpenAI or any AI tool company being discussed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (Aalborg University, academic research funding) has no financial interest in whether LLM-assisted code review is perceived positively or negatively."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an interview study investigating human perceptions, not evaluating a pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is an interview study investigating human perceptions, not evaluating a pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is an interview study investigating human perceptions, not evaluating a pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (e.g., OSF, AsPredicted). The study design and analysis plan were not committed to in advance."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D states: 'Ethical approvals, as per the authors' university requirements, were obtained prior to the study commencing.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides detailed demographics for all 20 participants: current role, experience level, gender, industry sector, programming language, assigned reviewers, and country."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes a detailed multi-stage screening process: programming task, critical-thinking question, problem-solving scenario, manual quality assessment, AI-content detection check, following Alami et al.'s prescreening guidelines."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study, not an experimental study with randomized conditions. All participants experienced both peer and LLM reviews in the same order."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative observational study. Blinding was not feasible given the study design — participants needed to know the source of reviews to discuss their perceptions of human vs. LLM feedback."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "76 participants agreed to interview but only 20 were interviewed. The paper does not explain the reduction from 76 to 20 or describe any attrition/dropout. For member checking, 19/20 responded (reported), but the initial attrition is unexplained."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study. There is no system whose inference cost is relevant."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study with no significant computational component."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [],
   "claims": [
     {
       "claim": "Engagement in code review is multi-dimensional, spanning cognitive, emotional, and behavioral responses.",
       "evidence": "Section IV-B presents evidence across all three dimensions with participant quotes: cognitive load varies with feedback clarity (P7, P14), emotional responses range from inspiration (P2) to feeling attacked (P6), and behavioral responses include reflection and revision (P2) or seeking clarification with defensiveness (P6).",
       "supported": "moderate"
     },
     {
       "claim": "LLM-assisted review is less emotionally taxing than peer review due to its consistent positive tone.",
       "evidence": "Multiple participants report preferring ChatGPT's tone: 'ChatGPT is also polite. So as long as it's polite, I don't really mind' (P8), 'I don't think it ever will be harsh... it just made it easier to kind of accept what it was telling me' (P10), 'ChatGPT in that regards is better. I reacted positively to its tone' (P6). Reported in Sect. IV-A and IV-B.",
       "supported": "moderate"
     },
     {
       "claim": "Cognitive load is higher when processing LLM-generated feedback compared to peer feedback.",
       "evidence": "P14 states: 'ChatGPT review will take more time and effort to analyze and to review, and others are more straightforward' (Sect. IV-A). P5 raises signal-to-noise concerns about LLM verbosity (Sect. IV-D). However, this is based on a small number of participant accounts.",
       "supported": "weak"
     },
     {
       "claim": "Software engineers use a similar sense-making process to evaluate and adopt feedback regardless of whether it comes from peers or LLMs.",
       "evidence": "P7 directly states: 'decision making, I still follow the same process I use with my peers' (Sect. IV-B). P5 describes line-by-line evaluation for both sources. However, the paper also notes that LLM feedback adoption is constrained by trust and context that differ from peer review.",
       "supported": "moderate"
     },
     {
       "claim": "LLM feedback adoption is constrained by trust in LLM abilities and lack of codebase context.",
       "evidence": "P20: 'if you go a bit deeper in more advanced topics, it doesn't really know what to do clearly... So I wouldn't trust ChatGPT' (Sect. IV-C). P18: 'AI might not be completely aware of what the context of the code you are writing' (Sect. IV-B). Multiple participants cite these constraints.",
       "supported": "moderate"
     },
     {
       "claim": "Most participants prefer combining peer and LLM reviews rather than replacing human reviewers entirely.",
       "evidence": "P13 suggests: 'I would use ChatGPT for a pre-review check... then put it out for review and then let a person look at it' (Sect. IV-B). P18 suggests combining to mitigate lack of context. The paper states 'most of our participants suggested combining both peers' and LLM's reviews' (Sect. IV-D).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["qualitative"],
   "key_findings": "Interview study of 20 software engineers reveals that code review engagement is multi-dimensional (cognitive, emotional, behavioral). LLM-assisted reviews reduce emotional burden through consistent positive tone but increase cognitive load due to verbosity and excessive detail. Engineers apply similar sense-making processes to both human and LLM feedback, but LLM adoption is constrained by trust and lack of codebase context. Most participants prefer combining peer and LLM reviews rather than full replacement of human reviewers.",
   "red_flags": [
     {
       "flag": "Small sample with broad claims",
       "detail": "N=20 Prolific-recruited participants with broad title and implications about 'software engineers' generally. The implications section (Sect. V) proposes organizational changes (EI training, feedback constructiveness programs) based on 20 interviews in an artificial setting."
     },
     {
       "flag": "No blinding to review source",
       "detail": "Participants knew which reviews came from ChatGPT and which from peers. This introduces expectation bias — participants may have responded differently to LLM reviews based on preconceptions about AI rather than the actual review content."
     },
     {
       "flag": "Order effect not controlled",
       "detail": "All participants first discussed peer reviews, then were shown the ChatGPT review. No counterbalancing was used. Order effects (novelty, fatigue, anchoring to peer reviews) could systematically bias the comparison."
     },
     {
       "flag": "Prolific recruitment may not represent professional software engineers",
       "detail": "Prolific participants who accept £60 interview studies may differ systematically from working software engineers. While the pre-screening was thorough, the authors acknowledge Prolific 'does not verify nor evaluate self-reported skills' (Sect. III-A)."
     },
     {
       "flag": "Artificial setting acknowledged but not mitigated",
       "detail": "Anonymous reviews between strangers differ fundamentally from real code review where developers have relationships, shared context, and team dynamics. The authors acknowledge this in Sect. VII but the study design cannot capture these critical aspects of professional code review."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["A. Fan", "B. Gokkaya", "M. Harman", "M. Lyubarskiy", "S. Sengupta", "S. Yoo", "J. M. Zhang"],
       "year": 2023,
       "relevance": "Comprehensive survey on LLMs for software engineering covering code generation, debugging, design, and code review."
     },
     {
       "title": "Information seeking using AI assistants",
       "authors": ["E. Al Haque", "C. Brown", "T. D. LaToza", "B. Johnson"],
       "year": 2024,
       "arxiv_id": "2408.04032",
       "relevance": "Identifies challenges with AI-generated content including excessive politeness, excessive detail, and lack of trustworthiness."
     },
     {
       "title": "Autonomy is an acquired taste: Exploring developer preferences for GitHub bots",
       "authors": ["A. Ghorbani", "N. Cassee", "D. Robinson", "A. Alami", "N. A. Ernst", "A. Serebrenik", "A. Wąsowski"],
       "year": 2023,
       "relevance": "Studies developer preferences for bot interactions in software engineering, finding varying trust levels and need for customizable interactions."
     },
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Z. Li", "S. Lu", "D. Guo", "N. Duan", "S. Jannu", "G. Jenks"],
       "year": 2022,
       "relevance": "Technical work on automating code review using pre-trained models, creating datasets of pull requests and code problems."
     },
     {
       "title": "A large-scale survey on the usability of AI programming assistants: Successes and challenges",
       "authors": ["J. T. Liang", "C. Yang", "B. A. Myers"],
       "year": 2024,
       "relevance": "Large-scale usability study of AI programming assistants finding developers get frustrated when AI lacks codebase context."
     },
     {
       "title": "Human-AI collaboration in software engineering: Lessons learned from a hands-on workshop",
       "authors": ["M. Hamza", "D. Siemon", "M. A. Akbar", "T. Rahman"],
       "year": 2024,
       "relevance": "Studies human-AI collaboration expectations in SE, finding engineers expect a 'collaborative partner' not merely a tool."
     },
     {
       "title": "LLaMA-Reviewer: Advancing code review automation with large language models through parameter-efficient fine-tuning",
       "authors": ["J. Lu", "L. Yu", "X. Li", "L. Yang", "C. Zuo"],
       "year": 2023,
       "relevance": "Technical work on fine-tuning LLMs specifically for code review automation."
     },
     {
       "title": "Using pre-trained models to boost code review automation",
       "authors": ["R. Tufano", "S. Masiero", "A. Mastropaolo", "L. Pascarella", "D. Poshyvanyk", "G. Bavota"],
       "year": 2022,
       "relevance": "Explores using pre-trained neural models to automate aspects of the code review process."
     },
     {
       "title": "Cobots in knowledge work: Human–AI collaboration in managerial professions",
       "authors": ["K. Sowa", "A. Przegalinska", "L. Ciechanowski"],
       "year": 2021,
       "relevance": "Studies human-AI collaboration in knowledge work, finding productivity increases when collaboration is enhanced."
     },
     {
       "title": "Explainable software bot contributions: Case study of automated bug fixes",
       "authors": ["M. Monperrus"],
       "year": 2019,
       "relevance": "Found that without explanation, it was difficult to get humans to accept automated code changes from bots."
     },
     {
       "title": "An empirical investigation of relevant changes and automation needs in modern code review",
       "authors": ["S. Panichella", "N. Zaugg"],
       "year": 2020,
       "doi": "10.1007/s10664-020-09870-3",
       "relevance": "Points out that code review tasks frequently change, and statically trained LLMs may miss evolving requirements."
     },
     {
       "title": "Constructive code review: Managing the impact of interpersonal conflicts in practice",
       "authors": ["P. Wurzel Goncalves", "J. SV Goncalves", "A. Bacchelli"],
       "year": 2024,
       "relevance": "Reports that negative feedback in code review causes interpersonal conflicts, directly relevant to the emotional engagement findings."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Provides insights for team leads on integrating AI code review, but no concrete tool, technique, or implementation guidance."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that LLM reviews reduce emotional load but increase cognitive load is mildly interesting but not deeply counterintuitive."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk, security, or safety concerns are raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims — findings are nuanced and balanced."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo to try — this is a qualitative interview study."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Mentions ChatGPT and GitHub Copilot but the study itself is from lesser-known academic institutions."
     }
   }
 }