ai-research-survey

scan.json (23712B)

---

 {
   "paper": {
     "title": "Evolving with AI: A Longitudinal Analysis of Developer Logs",
     "authors": [
       "Agnia Sergeyuk",
       "Eric Huang",
       "Dariia Karaeva",
       "Anastasiia Serova",
       "Yaroslav Golubev",
       "Iftekhar Ahmed"
     ],
     "year": 2026,
     "venue": "ICSE '26",
     "arxiv_id": "2601.10258",
     "doi": "10.1145/3744916.3787811"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Supplementary materials on Zenodo [35] include survey questionnaire, anonymized responses, interview script, and statistical analysis outputs, but no analysis source code or scripts are mentioned as released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Anonymized survey responses and complete statistical analysis outputs are publicly available via Zenodo [35]. Raw telemetry cannot be released due to confidentiality agreements, but aggregated data is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or software version details are provided for reproducing the statistical analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The statistical methodology is described but there are no runnable scripts or README with commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1-5 show shaded regions representing ±1 standard deviation from the monthly mean for all telemetry metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Mixed-effects linear models are used with p-values and a threshold of p < 0.05. Kolmogorov-Smirnov and Bartlett's tests are used to validate model assumptions. Multiple results report statistical significance (e.g., p < 0.001 for debugging, p = 0.03 for pastes)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect magnitudes are reported as regression coefficients with context: e.g., AI users +587 typed characters/month vs AI non-users +75, +102 deletions/month vs +7.6, +6.4 IDE activations vs -7.6."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for why 400 devices per group or 62 survey respondents were chosen. The sample sizes appear to be convenience-based."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation bands are shown in all telemetry figures (±1 SD shaded regions), and the mixed-effects model accounts for inter-device variability via random intercepts."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "400 AI non-users serve as a comparison group against 400 AI users, with trends analyzed for both groups over the same two-year period."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Both groups are drawn from the same time period (Oct 2022 - Oct 2024) and the same IDE platforms, making the comparison contemporary."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study of developer behavior, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five distinct telemetry metrics are used: typed characters, debugging instances, deletions, external pastes, and IDE window activations, plus seven Likert-scale survey questions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The study includes a 62-person survey capturing developer perceptions across all five dimensions, plus five semi-structured interviews providing qualitative context."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study analyzing real-world behavior, not a prediction task requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across five dimensions (productivity, code quality, code editing, code reuse, context switching), with separate analysis for AI users vs non-users in each."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Negative experiences are discussed, including P33's testimony about time wasted crafting prompts, P39's preference for taking responsibility over AI-generated code, and the finding that context switching increased for AI users."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: AI users delete significantly more code, context switching increases for AI users while decreasing for non-users, and a perception-behavior gap is documented where developers don't notice workflow changes."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that 'AI users produce substantially more code but also delete significantly more' and 'survey respondents report productivity gains and perceive minimal changes in other dimensions' are both directly supported by Section 4 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper's framing implies AI causes workflow changes (title: 'Evolving with AI', RQs ask 'How did the introduction of AI tools influence...'), but the study is observational with self-selected groups. Section 6 acknowledges 'These interpretations cannot be fully disentangled without experimental assignment to conditions,' but the overall framing still implies causation from an observational design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (External validity) explicitly states findings may not generalize beyond JetBrains IDEs and the specific AI assistant. They also acknowledge the AI user group may represent early adopters whose behaviors don't fully generalize."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Internal validity) substantively discusses self-selection bias: 'It is likely that they differ not only in AI usage but also in baseline motivation, experience, or task profiles.' They also discuss how the higher activity of AI users could reflect pre-existing characteristics rather than AI influence."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames each metric as a 'proxy' (e.g., 'As a proxy for productivity, we counted the number of typed characters') and acknowledges limitations: 'it captures only one specific facet of productivity — namely, code authoring — and does not account for other important activities such as debugging, design, or collaboration.' Section 3.2.1."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate any AI model's capability — it observes developer behavior. The AI assistant version is not specified, but the paper is studying user behavior, not model performance."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as a research methodology."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The mixed-effects linear model formula is explicitly provided (count_action ~ group × n_month + (1|id_device)), with p-value threshold of 0.05 and full statistical outputs available in supplementary materials."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.2 describes the data processing pipeline: monthly aggregation of timestamped action records per device, zero-filling for missing actions, and the sample selection criteria (activity in Oct 2022 and Oct 2024). 'No extensive filtering was applied to the data.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides substantial discussion organized by construct validity, internal validity, and external validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: developers using other AI tools may be misclassified as non-users, AI users may differ in baseline motivation/experience, telemetry from a single IDE vendor, and the proxy metrics capture only specific facets of each construct."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what results do not show: 'our findings may not generalize to all development environments or interface paradigms,' they cannot 'isolate causal effects,' and the distinction between groups is 'descriptive rather than explanatory.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw IDE telemetry logs cannot be released due to confidentiality agreements. Anonymized survey responses and aggregated statistical outputs are available, but the primary dataset (151M events) cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.1 describes telemetry collection (device IDs, event names, timestamps, metadata from IntelliJ IDEA, PyCharm, PhpStorm, WebStorm). Section 3.1.1 describes survey distribution (1,231 invitation emails to consented participant panel)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Survey recruitment is described: participants from an internal JetBrains panel who previously consented to research contact and self-identified as AI tool users. Interview participants selected for diversity across experience, geography, satisfaction, and usage patterns."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: telemetry logs → monthly aggregation per device → zero-filling → mixed-effects regression. Survey: 1,231 emails → 76 clicks → 67 responses → 62 after excluding non-AI users. Section 3.2.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments section is present. Three of six authors are affiliated with JetBrains (the company whose product is studied), but no explicit funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with JetBrains and JetBrains Research are clearly listed in the header. The collaboration with JetBrains is stated in Section 1."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The study was conducted 'in collaboration with JetBrains,' three authors are JetBrains employees, and the study evaluates JetBrains AI Assistant. JetBrains has a commercial interest in demonstrating the value of their AI tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model's capability on any benchmark. It is an observational study of developer behavior."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model benchmark evaluation is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No model benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the survey or the overall study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the study was 'conducted in line with our institution's ethical standards, adhering to the values and guidelines outlined in the ICC/ESOMAR International Code' — a marketing research ethics code, not IRB/ethics board approval."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.1 reports roles (56 developers, 16 team leads, 10 architects, 10 DevOps), experience levels (1-2 years to 16+ years), AI tools used, and usage duration."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Survey: self-identified AI tool users from JetBrains participant panel who consented to recontact. Telemetry: devices with activity in Oct 2022 and Oct 2024; AI users required consistent monthly JetBrains AI Assistant use from April-October 2024."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study, not an experimental study with random assignment to conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study; blinding is not applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Survey attrition is reported: 1,231 emails sent → 76 clicks → 67 complete responses → 62 after excluding non-AI users. 5 were excluded for not having used AI assistance."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study of developer behavior, not a system or method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study; computational budget is not a relevant concern."
       }
     }
   },
   "claims": [
     {
       "claim": "AI users produce substantially more code, with typed characters increasing at +587 characters/month vs +75 for non-users.",
       "evidence": "Mixed-effects linear model on 24 months of telemetry from 800 devices. Both trends statistically significant, interaction effect significant. Figure 1b, Section 4.1.",
       "supported": "strong"
     },
     {
       "claim": "AI users delete significantly more code (+102 deletions/month vs +7.6 for non-users).",
       "evidence": "Mixed-effects model on deletion events across the same 800 devices over 24 months. Both trends significant. Figure 3b, Section 4.3.",
       "supported": "strong"
     },
     {
       "claim": "82.3% of survey respondents perceive productivity gains from AI tools.",
       "evidence": "62 survey responses on 5-point Likert scale. Figure 1a, Section 4.1.",
       "supported": "moderate"
     },
     {
       "claim": "Developers perceive minimal changes in code quality, editing, reuse, and context switching, despite measurable telemetry differences.",
       "evidence": "Survey results show ~50% 'no change' for code quality, editing, reuse, and context switching (Figures 2a-5a), while telemetry shows significant trends for AI users in deletions, pastes, and IDE activations. Table 1, Section 5.2.",
       "supported": "strong"
     },
     {
       "claim": "Context switching increases for AI users (+6.4 IDE activations/month) while decreasing for non-users (-7.6/month).",
       "evidence": "Mixed-effects model on IDE window activation events. Both trends statistically significant with opposing directions. Figure 5b, Section 4.5.",
       "supported": "strong"
     },
     {
       "claim": "AI does not uniformly reduce developer effort; it redistributes effort across more fragmented workflows.",
       "evidence": "Synthesis of all five dimensions: more typing + more deletion + more context switching = redistributed, not reduced effort. Section 5.1-5.2.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "observational",
     "qualitative"
   ],
   "key_findings": "A two-year longitudinal study of 800 developers' IDE telemetry combined with a 62-person survey reveals that AI tool adoption is associated with substantially increased code output (+587 chars/month vs +75 for non-users) but also significantly more deletions (+102/month vs +7.6). Developers perceive strong productivity gains (82.3%) but report minimal changes in other dimensions, despite telemetry showing increased context switching and external paste activity among AI users. The perception-behavior gap suggests AI silently restructures development workflows in ways developers don't consciously recognize.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Three of six authors are JetBrains employees, and the study evaluates the impact of JetBrains AI Assistant using JetBrains telemetry data. While the paper acknowledges this in threats to validity, no formal conflict-of-interest statement is included."
     },
     {
       "flag": "Self-selection bias in group comparison",
       "detail": "AI users and non-users are self-selected, not randomly assigned. AI users are likely more active developers to begin with — the paper acknowledges 'AI users are generally more active in the IDE' but still frames findings as AI-mediated changes. The causal framing (title, RQs) overreaches the observational design."
     },
     {
       "flag": "Survey recruitment bias",
       "detail": "Survey respondents were recruited from JetBrains' internal participant panel of users who previously self-identified as AI tool users. This is a convenience sample biased toward JetBrains users and AI enthusiasts, yielding only 62 responses from 1,231 invitations (5% response rate)."
     },
     {
       "flag": "Misclassification of AI non-users",
       "detail": "Non-users are defined as those who never used JetBrains AI Assistant. They may use ChatGPT, Copilot, or other external AI tools extensively. The paper acknowledges this but still labels them 'AI non-users.'"
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the impact of early-2025 AI on experienced open-source developer productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT finding that AI assistants actually increase task completion time by 19%, directly contradicting productivity claims."
     },
     {
       "title": "The impact of AI on developer productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Key Copilot productivity study claiming 50%+ improvement in task completion time."
     },
     {
       "title": "Reading between the lines: Modeling user behavior and costs in AI-assisted programming",
       "authors": ["Hussein Mozannar", "Gagan Bansal", "Adam Fourney", "Eric Horvitz"],
       "year": 2024,
       "relevance": "Found developers spend over 50% of time evaluating/editing AI-generated output, relevant to productivity measurement."
     },
     {
       "title": "A large-scale survey on the usability of AI programming assistants: Successes and challenges",
       "authors": ["Jenny T Liang", "Chenyang Yang", "Brad A Myers"],
       "year": 2024,
       "relevance": "Large-scale survey on AI assistant usability, finding some developers actively avoid AI tools."
     },
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["Albert Ziegler", "Eirini Kalliamvakou", "X Alice Li"],
       "year": 2022,
       "relevance": "Documents disconnect between perceived and actual productivity with code completion tools."
     },
     {
       "title": "The impact of generative AI on collaborative open-source software development: Evidence from GitHub Copilot",
       "authors": ["Fangchen Song", "Ashish Agarwal", "Wen Wen"],
       "year": 2024,
       "arxiv_id": "2410.02091",
       "relevance": "Reports no significant quality change despite productivity improvements with Copilot."
     },
     {
       "title": "Using AI-based coding assistants in practice: State of affairs, perceptions, and ways forward",
       "authors": ["Agnia Sergeyuk", "Yaroslav Golubev", "Timofey Bryksin", "Iftekhar Ahmed"],
       "year": 2025,
       "relevance": "Prior survey of AI coding assistant usage by some of the same authors."
     },
     {
       "title": "Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": ["Priyan Vaithilingam", "Tianyi Zhang", "Elena L Glassman"],
       "year": 2022,
       "relevance": "Found Copilot did not consistently reduce task time, documenting expectation-experience gap."
     },
     {
       "title": "Are large language models a threat to digital public goods? Evidence from activity on Stack Overflow",
       "authors": ["Maria del Rio-Chanona", "Nadzeya Laurentsyeva", "Johannes Wachs"],
       "year": 2023,
       "arxiv_id": "2307.07367",
       "relevance": "Documents StackOverflow activity drop post-ChatGPT, relevant to code reuse patterns."
     },
     {
       "title": "The SPACE of developer productivity: There's more to it than you think",
       "authors": ["Nicole Forsgren", "Margaret-Anne Storey", "Chandra Maddila"],
       "year": 2021,
       "relevance": "Foundational framework for developer productivity measurement, relevant to proxy-outcome distinction."
     }
   ]
 }