ai-research-survey

scan.json (23866B)

---

 {
   "paper": {
     "title": "AI in Software Engineering: Perceived Roles and Their Impact on Adoption",
     "authors": ["Ilya Zakharov", "Ekaterina Koshchenko", "Agnia Sergeyuk"],
     "year": 2025,
     "venue": "FSE Companion '25",
     "arxiv_id": "2504.20329",
     "doi": "10.1145/3696630.3730563"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or analysis scripts are released. The paper mentions a survey questionnaire on Zenodo (DOI: 10.5281/zenodo.14973853) but no analysis code repository is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'All anonymized data analyzed in this study is available upon request' (Section 4), which counts as NO per schema rules. The survey instrument is available on Zenodo, but the underlying response data is not publicly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification is provided. The paper mentions using Python packages 'factor-analyzer' and SciPy but does not provide version details for the environment, requirements.txt, or any reproduction environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The analysis methods are described at a high level (factor analysis, Pearson correlations) but there are no scripts, commands, or detailed reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Table 1 reports correlation coefficients with significance levels (p-value stars) but no confidence intervals for the correlations."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Statistical significance is reported for Pearson correlation coefficients in Table 1, with significance levels marked as *** p<0.001, ** p<0.01, * p<0.05."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pearson correlation coefficients (r values) are reported in Table 1 (e.g., r=0.59, r=0.41, r=0.34 for PU correlations), which serve as effect sizes. Factor loadings are also reported with specific values (e.g., 'problem solver' loading=0.599, 'advisor' loading=0.662)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the sample size of N=102 survey participants or N=38 interview participants is provided. No power analysis is discussed. For factor analysis with 14 role variables, N=102 is arguably on the lower end, but this is not addressed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviations, or spread measures are reported for the TAM scale scores (PU, PEU) or role counts. Only correlation coefficients and factor loadings are presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines or comparisons to prior work measuring similar constructs are included. The paper does not compare its correlation findings against prior studies of role attribution or TAM."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so contemporaneity cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple outcome variables are used: Perceived Usefulness (PU), Perceived Ease of Use (PEU), and number of AI tools tried. Factor analysis also extracts two factors (Expert Roles, Support Roles)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not produce system outputs that need human evaluation. It is a survey study about user perceptions, not an evaluation of an AI system's outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study, not a machine learning or prediction task. There is no test set concept applicable here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-role frequency counts are reported (e.g., assistant n=70, tool n=69), and per-factor breakdowns are provided through factor analysis (Expert Roles vs. Support Roles) with individual role loadings."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or limitations of the role attribution framework are discussed. The paper does not examine cases where the model breaks down or where role attribution does not predict acceptance."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that coding experience did not significantly correlate with any of the other variables (Table 1, e.g., r=-0.07, -0.20, 0.18, -0.05, -0.11, 0.18, all non-significant), which is a null/negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims two Mental Models (tool vs. teammate), two factor groupings (Support Roles, Expert Roles), and positive correlation between role count and PU/PEU. All are supported by the qualitative analysis (Section 4.1), factor analysis (Section 4.2), and Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states that 'diverse conceptualizations enhance AI adoption,' which is a causal claim ('enhance'). The evidence is correlational (Pearson r from a cross-sectional survey). The study design does not support causal inference — it cannot distinguish whether assigning more roles causes higher acceptance or vice versa."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about 'developers' and 'AI4SE tools' generally, but the sample is drawn from a JetBrains user panel — people who consented to participate in JetBrains user studies. This is a convenience sample from a single company's ecosystem, but the paper does not bound its generalizations to this population."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed correlations are discussed. For example, the correlation between number of roles and PU/PEU could be driven by a confound such as general AI enthusiasm or engagement level, but this possibility is not addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use or evaluate any AI models. It is a survey study about developer perceptions of AI tools in general."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No LLM prompting is used in this study. It is a human survey and interview study."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Factor analysis parameters are reported: varimax rotation, factor loading threshold of >0.4 (Section 3). The Python packages used (factor-analyzer, SciPy) are named."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a survey/interview study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Data preprocessing steps are not documented. The paper does not describe how survey responses were cleaned, whether any responses were excluded, or how the Revised TAM Questionnaire responses were processed beyond stating that 'each scale score was obtained by averaging the relevant items.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion (Section 6) mentions future research directions but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. Specific threats such as the JetBrains panel sampling bias, the small N for factor analysis, or the cross-sectional design are not mentioned."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what its results do NOT show (e.g., that correlations do not imply causation, that the sample may not represent developers outside the JetBrains ecosystem)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data is not available. The paper states 'All anonymized data analyzed in this study is available upon request' (Section 4), which is not publicly accessible for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: interviews were conducted in spring 2024 with 38 programmers (details in reference [15]), and an online survey was conducted in January 2025 with 102 participants recruited from a JetBrains curated panel (Section 3)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Recruitment methods are described: 'The link to the survey was sent to a curated list of people who had given their consent to participate in user studies conducted by JetBrains' (Section 3). Interview participants were from a prior study [15]."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The data pipeline is not fully documented. It is unclear how many people received the survey link vs. the 102 who completed it (response rate unknown), whether any responses were excluded, or how the qualitative coding process worked for the interview re-analysis."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. All three authors are affiliated with JetBrains Research, a company that makes developer tools, but no funding acknowledgment section is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list JetBrains Research as their affiliation, which is clearly stated in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "JetBrains is a developer tools company that sells AI-powered development tools. The authors are JetBrains Research employees studying adoption of AI-powered development tools. The funder (JetBrains) has a direct commercial interest in the outcome — findings that role attribution increases AI tool adoption support the company's business case."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The authors work for a company that sells the tools being studied, but this conflict is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate any pre-trained model's capability on a benchmark. It is a survey/interview study about developer perceptions."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model evaluation on benchmarks is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. No link to OSF, AsPredicted, or any pre-registration platform is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned. The paper states the study 'was carried out according to JetBrains' ethical standards, adhering to the values and guidelines outlined in the ICC/ESOMAR International Code' (Section 3), but this is an industry marketing research code, not an institutional ethics board review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Participant demographics are reported: coding experience breakdown (3 no experience, 9 <1 year, 12 1-2 years, 21 3-5 years, 19 6-10 years, 14 11-15 years, 24+ 16+ years), and job types for non-coders (management, digital design, online advertising, molecular biology) in Section 4.2."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for the survey. Participants were drawn from 'a curated list of people who had given their consent to participate in user studies conducted by JetBrains' but no screening criteria or exclusion rules are described."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experimental study with treatment conditions. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experimental study. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "Attrition is not reported. The paper states '102 participants fully completed the survey' but does not mention how many started and did not finish, or the total number of people invited."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study, not a system or method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study with minimal computational requirements. Compute budget is not applicable."
       }
     }
   },
   "claims": [
     {
       "claim": "Developers conceptualize AI-powered Development Tools along two primary dimensions: as inanimate tools or as human-like teammates.",
       "evidence": "Qualitative analysis of 38 interviews showed ~80% referred to AI as a machine/software/tool, while ~20% defined AI in human terms using words like colleague, companion, or assistant (Section 4.1).",
       "supported": "moderate"
     },
     {
       "claim": "AI roles can be grouped into two factors: Support Roles (assistant, reference guide, tool) and Expert Roles (problem solver, advisor, reviewer).",
       "evidence": "Factor analysis with varimax rotation on survey data from 102 participants, with factor loadings reported (e.g., assistant=0.657, problem solver=0.599, advisor=0.662, reviewer=0.691) in Section 4.2.",
       "supported": "moderate"
     },
     {
       "claim": "Assigning multiple roles to AI correlates positively with Perceived Usefulness and Perceived Ease of Use.",
       "evidence": "Table 1 shows correlations: total AI roles with PU r=0.59 (p<0.001), with PEU r=0.56 (p<0.001). Expert Roles with PU r=0.41 (p<0.001), PEU r=0.42 (p<0.001). Support Roles with PU r=0.34 (p<0.001), PEU r=0.27 (p<0.01).",
       "supported": "moderate"
     },
     {
       "claim": "Diverse conceptualizations enhance AI adoption.",
       "evidence": "Based on correlational evidence from Table 1. The causal language ('enhance') is not supported by the cross-sectional survey design, which cannot rule out reverse causation or confounding.",
       "supported": "weak"
     },
     {
       "claim": "Coding experience does not significantly correlate with AI role attribution or technology acceptance.",
       "evidence": "Table 1 shows non-significant correlations between coding experience and all other variables (r ranging from -0.20 to 0.18, none reaching p<0.05).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["qualitative", "observational"],
   "key_findings": "Developers conceptualize AI-powered development tools along two dimensions: as inanimate tools (~80%) or human-like teammates (~20%). Factor analysis of survey data from 102 participants reveals two role groupings: Support Roles (assistant, reference guide, tool) and Expert Roles (advisor, problem solver, reviewer). Assigning more roles to AI correlates positively with Perceived Usefulness (r=0.59) and Perceived Ease of Use (r=0.56), while coding experience shows no significant relationship with role attribution or acceptance.",
   "red_flags": [
     {
       "flag": "Conflict of interest: JetBrains employees studying JetBrains tool users",
       "detail": "All authors are from JetBrains Research, a company that sells AI-powered development tools. The survey sample is drawn from JetBrains' own user panel. Findings that role attribution increases AI tool adoption directly serve the company's commercial interest. This conflict is not acknowledged."
     },
     {
       "flag": "Convenience sample presented as general",
       "detail": "The survey sample is drawn from 'a curated list of people who had given their consent to participate in user studies conducted by JetBrains.' This is a self-selected group of JetBrains users, likely more engaged with development tools than average developers, but the paper generalizes to 'developers' broadly."
     },
     {
       "flag": "Causal language from correlational data",
       "detail": "The abstract claims 'diverse conceptualizations enhance AI adoption' and the discussion states role attributions 'directly influence technology adoption decisions.' These are causal claims derived from cross-sectional correlational data, which cannot establish causality."
     },
     {
       "flag": "No limitations section",
       "detail": "A 5-page short paper at a top venue discusses no limitations, threats to validity, or scope boundaries. The cross-sectional design, small N for factor analysis, JetBrains panel bias, and correlational-vs-causal issues are all unaddressed."
     },
     {
       "flag": "Small sample for factor analysis",
       "detail": "N=102 for factor analysis with 14 role variables is on the lower end of recommended sample sizes. Rules of thumb suggest 5-10 participants per variable (70-140 needed), and some methodologists recommend N>200. The adequacy of the sample for factor analysis is not discussed."
     },
     {
       "flag": "Suspiciously high PU-PEU correlation",
       "detail": "PU and PEU correlate at r=0.94, which is unusually high for two purportedly distinct constructs. This raises concerns about discriminant validity of the TAM scales in this sample, but is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Beyond accuracy: The role of mental models in human-AI team performance",
       "authors": ["Gagan Bansal", "Besmira Nushi", "Ece Kamar", "Walter S Lasecki", "Daniel S Weld", "Eric Horvitz"],
       "year": 2019,
       "relevance": "Examines how mental models of AI affect human-AI team performance, directly relevant to understanding AI tool adoption."
     },
     {
       "title": "What Guides Our Choices? Modeling Developers' Trust and Behavioral Intentions Towards GenAI",
       "authors": ["Rudrajit Choudhuri", "Bianca Trinkenreich", "Rahul Pandita", "Eirini Kalliamvakou"],
       "year": 2024,
       "arxiv_id": "2409.04099",
       "relevance": "Models developer trust and behavioral intentions toward generative AI tools, directly relevant to AI adoption in software engineering."
     },
     {
       "title": "AI tool use and adoption in software development by individuals and organizations: a grounded theory study",
       "authors": ["Ze Shi Li", "Nowshin Nawar Arony", "Ahmed Musa Awon", "Daniela Damian", "Bowen Xu"],
       "year": 2024,
       "arxiv_id": "2406.17325",
       "relevance": "Grounded theory study of AI tool adoption in software development, complementary methodology to the surveyed paper."
     },
     {
       "title": "Navigating the complexity of generative AI adoption in software engineering",
       "authors": ["Daniel Russo"],
       "year": 2024,
       "relevance": "Studies generative AI adoption complexity in software engineering using TAM-related constructs."
     },
     {
       "title": "The Design Space of in-IDE Human-AI Experience",
       "authors": ["A. Sergeyuk", "E. Koshchenko", "I. Zakharov", "T. Bryksin", "M. Izadi"],
       "year": 2024,
       "arxiv_id": "2410.08676",
       "relevance": "Companion qualitative study (same group) on human-AI experience in IDEs; source of the interview data reanalyzed in this paper."
     },
     {
       "title": "Influencing human-AI interaction by priming beliefs about AI can increase perceived trustworthiness, empathy and effectiveness",
       "authors": ["Pat Pataranutaporn", "Ruby Liu", "Ed Finn", "Pattie Maes"],
       "year": 2023,
       "doi": "10.1038/s42256-023-00720-7",
       "relevance": "Demonstrates how priming beliefs about AI affects trust and interactions, relevant to mental models and AI adoption."
     },
     {
       "title": "Mutual theory of mind for human-AI communication",
       "authors": ["Qiaosi Wang", "Ashok K. Goel"],
       "year": 2022,
       "relevance": "Proposes framework for mutual theory of mind in human-AI interaction, theoretical foundation for studying AI role attribution."
     },
     {
       "title": "Mutual theory of mind in human-AI collaboration: An empirical study with LLM-driven AI agents in a real-time shared workspace task",
       "authors": ["Shao Zhang", "Xihuai Wang", "Wenhao Zhang"],
       "year": 2024,
       "arxiv_id": "2409.08811",
       "relevance": "Empirical study of mutual theory of mind with LLM agents, tests whether MToM frameworks improve human-AI collaboration."
     }
   ]
 }