ai-research-survey

scan.json (27252B)

---

 {
   "paper": {
     "title": "Beyond the Commit: Developer Perspectives on Productivity with AI Coding Assistants",
     "authors": [
       "Valerie Chen",
       "Jasmyn He",
       "Behnjamin Williams",
       "Jason Valentino",
       "Ameet Talwalkar"
     ],
     "year": 2026,
     "venue": "ICSE-SEIP '26",
     "arxiv_id": "2602.03593",
     "doi": "10.1145/3786583.3786848"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, Zenodo archive, or any code/data artifact link is provided in the paper. The survey instrument and interview protocol are not released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The survey data and interview transcripts are not released. The paper notes the survey was conducted within BNY Mellon and certain questions are omitted 'to protect the confidentiality of company metrics and goals.' No anonymized dataset is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. The analysis methods (Pearson correlation, thematic coding) are described conceptually but no analysis scripts or software environment details are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The survey was administered via the DX platform at BNY Mellon, a proprietary setting, and no steps for reproducing the analysis are included."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The survey results are reported as percentages (e.g., '86% are either satisfied or very satisfied') without confidence intervals or error bars. The correlation is reported as r=0.34 with p<0.0001 but no confidence interval for the correlation coefficient."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports a Pearson Correlation Score of r=0.34 with p<0.0001 for the relationship between satisfaction and time savings (Section 4.1). This is a significance test for the correlation."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The Pearson correlation r=0.34 is itself an effect size measure for the relationship between satisfaction and time savings. The paper characterizes it as 'a positive but weak correlation,' providing context for interpretation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The survey sample of N=2989 is large but not justified through power analysis. The interview sample of N=11 is not justified with any formal reasoning about saturation or adequacy for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any quantitative results. The survey results are presented only as aggregate percentages and a single correlation value."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares its identified factors against existing frameworks (SPACE, DORA) and prior studies in Table 3, showing which factors are and are not captured by existing work."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison frameworks and prior studies cited are contemporary, including SPACE (2021), DORA (2018), and recent empirical studies from 2022-2025 (Table 3)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a mixed-methods qualitative/survey study, not a system with components to ablate. There is no system or tool being proposed that could be decomposed."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly uses two survey metrics—general satisfaction and perceived time savings—and demonstrates that they capture different aspects of productivity (r=0.34 weak correlation). The qualitative component identifies six additional factors."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not produce system outputs that require human evaluation. It is itself a study of human perspectives. Human evaluation of system outputs is not applicable here."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning or benchmark study. There is no test set or training/test split applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 (right panel) provides a fine-grained breakdown of satisfaction vs. time savings categories. Table 1 breaks down interview participants by seniority, role, and department. Figure 2 maps use cases to impact factors."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where AI tools fail: hallucinations ('asking it to fix it does not work and just returns the same wrong answer'), limitations with certain languages ('does not work as well for C#'), and difficulties with refactoring tasks (Section 4.3.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: 60% of engineers report less than one hour of time savings, satisfaction and time savings are only weakly correlated (r=0.34), AI may erode junior developer skills (Factor 5), and refactoring use cases are particularly challenging (Section 4.3.2). Figure 2 explicitly marks negative impacts."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: 'survey results expose conflicting perspectives' is backed by the weak correlation finding (r=0.34); 'six distinct factors' are identified in Section 4.2; 'long-term metrics like technical expertise and ownership' are detailed in Section 4.2.3. No overclaiming detected."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to avoid strong causal claims. It uses language like 'our findings demonstrate that a multifaceted approach is needed' and 'these results suggest that developers can be satisfied...without necessarily getting significant time savings.' The claims are correlational and descriptive, matching the observational study design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 (Threats to Validity) explicitly states 'The surveys and interviews conducted in this work were limited to one company' and acknowledges results 'may not generalize to all other organizations or to independent developers.' The scope is bounded to BNY Mellon and GitHub Copilot."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for the satisfaction-time savings disconnect: hallucinations, language-specific limitations, and the possibility that satisfaction reflects factors beyond time savings. Section 5.3 discusses that the findings may reflect BNY Mellon's specific adoption stage rather than general patterns."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not run experiments with LLMs. It studies developers' experiences with GitHub Copilot as a black-box tool. There are no model versions to specify for the study itself."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as part of its methodology. It studies how developers use GitHub Copilot but does not run any LLM experiments itself."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLM experiments are conducted. The study is a survey and interview study, not a computational experiment with hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper studies developer experiences with GitHub Copilot as a black-box third-party tool."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that 8000 total engineers completed the survey but only 2989 responded to the AI-specific questions, and the correlation analysis uses N=2754 who answered both questions. However, the filtering criteria for going from 8000 to 2989 are vague ('Engineers opted in to answering questions based on whether it was relevant to them'). The criteria for the interview coding process are described at a high level but lack specific detail about how codes were developed and refined."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 is titled 'Threats to Validity' and provides a substantive discussion of limitations spanning an entire paragraph."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 identifies specific threats: limitation to one company (BNY Mellon), access only to GitHub Copilot (not other AI coding assistants), exclusion of agentic workflows, varied response rates across subgroups, and factors may not be fully comprehensive. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 explicitly states boundaries: results limited to one company, only GitHub Copilot was assessed, agentic workflows 'were out of scope for this work,' and the factors 'may not be fully comprehensive.' These are specific scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw survey data or interview transcripts are released. The company setting likely prevents this due to confidentiality, but the data cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the survey: conducted via the DX platform, targeting engineers who actively commit code at BNY Mellon, with specific questions listed. Section 3.2 describes the interview process: 30-45 minute semi-structured interviews via Teams, with the protocol topics listed and participant selection criteria specified."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Survey recruitment: 'we leveraged the large pool of engineers throughout BNY Mellon who actively commit code' with opt-in participation. Interview recruitment: purposive sampling based on three criteria (seniority, business sector, development role), filtered from an internal list, contacted via Teams, supplemented by snowball sampling (Section 3.2)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from 8000 total survey respondents to 2989 AI-related respondents to 2754 used in correlation analysis is mentioned but the filtering steps are not clearly documented (what caused the drop from 2989 to 2754 is explained, but the 8000→2989 drop relies on opt-in relevance with no formal criteria). The qualitative coding process is described at a high level ('inductive open coding,' 'iterating multiple times') but specific stages and counts are not provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. Three of the five authors are affiliated with BNY Mellon, but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Valerie Chen and Ameet Talwalkar at Carnegie Mellon University; Jasmyn He, Behnjamin Williams, and Jason Valentino at BNY Mellon. The company affiliation is prominent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three authors are BNY Mellon employees and the study was conducted at BNY Mellon using their engineering population. BNY Mellon has a commercial interest in demonstrating value from their GitHub Copilot deployment. No disclosure is made about whether the funder (BNY Mellon, implicitly) has independence from the study outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosures are present in the paper. Absence of disclosure is not absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark. It is a survey and interview study of developer experiences."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark. Contamination is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark. Contamination is not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. No link to OSF, AsPredicted, or any registry is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned in the paper, despite conducting a survey of 2989 employees and 11 interviews."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports interview participant demographics: seniority level (early career, mid career, management), developer role (backend, full stack, frontend), and department function. Survey respondents are described as 'engineers who actively commit code' with access to GitHub Copilot. However, detailed demographics for survey respondents (gender, experience years, etc.) are not provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Interview participants were selected using three criteria: 'different levels of seniority,' 'business sector,' and 'varying roles in development' (Section 3.2). Survey participants were engineers who 'actively commit code' and had access to GitHub Copilot. These constitute stated inclusion criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment and control conditions. It is a cross-sectional survey and qualitative interview study. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study. There are no conditions to blind participants to. Blinding is not applicable for a survey and interview study."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 8000 engineers completed the broader survey, 2989 answered the AI-specific questions, and 2754 answered both questions for the correlation analysis. For interviews, they note 'the response rate varied' across subgroups and all 11 completed the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey and interview study, not a computational method. There is no inference cost applicable to the paper's own methodology."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey and interview study with no significant computational component. Compute budget is not applicable."
       }
     }
   },
   "claims": [
     {
       "claim": "86% of developers are satisfied or very satisfied with GitHub Copilot, but around 60% report less than one hour of time savings per week.",
       "evidence": "Figure 1 left panel shows 50% very satisfied and 36% satisfied. Figure 1 middle panel shows 29% save 1-30 min, 23% save 31-60 min, and 8% report no time savings (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "The correlation between satisfaction and perceived time savings is weak (r=0.34, p<0.0001), demonstrating that different productivity metrics can give conflicting signals.",
       "evidence": "Section 4.1 reports Pearson Correlation Score of r=0.34 with N=2754, p<0.0001. Figure 1 right panel shows nearly 400 developers very satisfied despite saving only 30 minutes per week.",
       "supported": "strong"
     },
     {
       "claim": "Six factors capture the breadth of AI coding assistant productivity impacts: self-sufficiency, frustration/cognitive load, task completion rate, ease of peer review, technical expertise, and ownership of work.",
       "evidence": "Section 4.2 details these six factors derived from 11 semi-structured interviews, with supporting quotes from participants. Table 2 provides operationalization questions for each factor.",
       "supported": "moderate"
     },
     {
       "claim": "Long-term factors (technical expertise and ownership of work) are largely unaddressed in existing frameworks like SPACE and DORA and in prior empirical studies.",
       "evidence": "Table 3 shows that Factors 5 (long-term technical expertise) and 6 (ownership of work) have 'No' for SPACE/DORA coverage and 'N/A' for prior studies (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "The impact of GitHub Copilot on productivity factors varies by use case, with different use cases showing different positive and negative impacts across the six factors.",
       "evidence": "Figure 2 maps three use cases (implementing new features, improving existing code, generating tests/documentation) to the six factors, showing varied positive and negative impacts. Sections 4.3.1-4.3.3 provide qualitative support.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "observational"
   ],
   "key_findings": "A survey of 2989 developers at BNY Mellon reveals that 86% are satisfied with GitHub Copilot but about 60% report less than one hour of weekly time savings, with only a weak correlation (r=0.34) between satisfaction and time savings. Semi-structured interviews with 11 developers identify six productivity factors spanning development (self-sufficiency, cognitive load), deployment (task completion, peer review), and long-term impacts (technical expertise, ownership of work). The study highlights that long-term factors like skill development and code ownership are largely absent from existing productivity frameworks (SPACE, DORA) and prior empirical studies of AI coding assistants.",
   "red_flags": [
     {
       "flag": "Small interview sample for broad claims",
       "detail": "Only 11 interviews from a single company are used to derive a six-factor productivity framework presented as broadly applicable. While qualitative research can use small samples, the diversity of perspectives within BNY Mellon may not capture the full range of AI productivity dimensions across different organizations, team sizes, or development cultures."
     },
     {
       "flag": "Company-affiliated authors studying company's tool deployment",
       "detail": "Three of five authors are BNY Mellon employees, and the study evaluates productivity of AI tools deployed at BNY Mellon. While the paper does report mixed/negative findings, no conflict of interest statement or funding disclosure is present."
     },
     {
       "flag": "Self-reported time savings rather than measured",
       "detail": "The paper acknowledges this limitation: 'we considered the use of direct objective metrics, like acceptance rates or chat acceptance rates, that do not rely on self-reports, but were unable to reliably obtain these metrics.' Self-reported time savings are known to be unreliable, as the paper itself notes (Section 5.1)."
     },
     {
       "flag": "No IRB or ethics approval mentioned",
       "detail": "The study collects survey data from 2989 employees and conducts 11 interviews at their employer, but no IRB or ethics board approval is mentioned. This raises questions about participant protections, especially given the employer-employee dynamic."
     }
   ],
   "cited_papers": [
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2306.08516",
       "relevance": "RCT studying causal impact of GitHub Copilot on developer productivity, a key baseline for this survey's productivity claims."
     },
     {
       "title": "Measuring GitHub Copilot's Impact on Productivity",
       "authors": ["Albert Ziegler", "Eirini Kalliamvakou", "X. Alice Li", "Andrew Rice", "Devon Rifkin", "Shawn Simister", "Ganesh Sittampalam", "Edward Aftandilian"],
       "year": 2024,
       "relevance": "Empirical study of GitHub Copilot's productivity impact using self-reported efficiency and flow measures."
     },
     {
       "title": "The SPACE of Developer Productivity: There's more to it than you think",
       "authors": ["Nicole Forsgren", "Margaret-Anne Storey", "Chandra Maddila", "Thomas Zimmermann", "Brian Houck", "Jenna Butler"],
       "year": 2021,
       "relevance": "Foundational developer productivity framework (SPACE) that this paper extends for AI coding assistants."
     },
     {
       "title": "Better Together? An Evaluation of AI-Supported Code Translation",
       "authors": ["Justin D. Weisz", "Michael Muller", "Steven I. Ross"],
       "year": 2022,
       "relevance": "Evaluates variation in developer reliance on AI assistance, relevant to the self-sufficiency factor identified in this paper."
     },
     {
       "title": "The RealHumanEval: Evaluating Large Language Models' Abilities to Support Programmers",
       "authors": ["Hussein Mozannar", "Valerie Chen", "Mohammed Alsobay"],
       "year": 2024,
       "arxiv_id": "2404.02806",
       "relevance": "Evaluates LLMs' ability to support programmers through human evaluation, directly relevant to AI productivity measurement."
     },
     {
       "title": "An empirical evaluation of GitHub copilot's code suggestions",
       "authors": ["Nhan Nguyen", "Sarah Nadi"],
       "year": 2022,
       "relevance": "Empirical evaluation of AI code quality and reviewability, relevant to the peer review factor identified in this paper."
     },
     {
       "title": "The Impact of Generative AI on Collaborative Open-Source Software Development: Evidence from GitHub Copilot",
       "authors": ["Fangchen Song", "Ashish Agarwal", "Wen Wen"],
       "year": 2024,
       "arxiv_id": "2401.01812",
       "relevance": "Field study measuring objective impacts of GitHub Copilot on open-source development, complementary evidence to this survey's subjective measures."
     },
     {
       "title": "Modeling User Behavior and Costs in AI-Assisted Programming",
       "authors": ["Hussein Mozannar", "Gagan Bansal", "Adam Fourney", "Eric Horvitz"],
       "year": 2024,
       "doi": "10.1145/3613904.3641936",
       "relevance": "Models developer behavior and cognitive costs with AI assistants, directly relevant to the cognitive load factor."
     },
     {
       "title": "A large-scale survey on the usability of ai programming assistants: Successes and challenges",
       "authors": ["Jenny T Liang", "Chenyang Yang", "Brad A Myers"],
       "year": 2024,
       "relevance": "Large-scale survey on AI programming assistant usability, complementary methodology to this paper's BNY Mellon survey."
     },
     {
       "title": "Is GitHub copilot a substitute for human pair-programming? an empirical study",
       "authors": ["Saki Imai"],
       "year": 2022,
       "doi": "10.1145/3510454.3522684",
       "relevance": "Empirical study comparing AI assistance to pair programming in terms of productivity, relevant to task completion metrics."
     },
     {
       "title": "What's DAT? Three Case Studies of Measuring Software Development Productivity at Meta With Diff Authoring Time",
       "authors": ["Moritz Beller", "Amanda Park", "Karim Nakad"],
       "year": 2025,
       "arxiv_id": "2503.10977",
       "relevance": "Introduces Diff Authoring Time as an objective productivity metric at Meta, relevant to operationalizing AI productivity measurement."
     },
     {
       "title": "Code with me or for me? how increasing ai automation transforms developer workflows",
       "authors": ["Valerie Chen", "Ameet Talwalkar", "Robert Brennan", "Graham Neubig"],
       "year": 2025,
       "arxiv_id": "2507.08149",
       "relevance": "Studies how AI automation levels affect developer workflows, extending the productivity framework from this paper to agentic tools."
     },
     {
       "title": "Examining the use and impact of an ai code assistant on developer productivity and experience in the enterprise",
       "authors": ["Justin D Weisz", "Shraddha Vijay Kumar", "Michael Muller"],
       "year": 2025,
       "relevance": "Enterprise study of AI code assistant impact on developer productivity, directly comparable to this paper's BNY Mellon study."
     }
   ]
 }