ai-research-survey

scan.json (26499B)

---

 {
   "paper": {
     "title": "Code with Me or for Me? How Increasing AI Automation Transforms Developer Workflows",
     "authors": ["Valerie Chen", "Ameet Talwalkar", "Robert Brennan", "Graham Neubig"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.08149"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides analysis scripts at https://github.com/valeriechen/copilot-agent-comparison (footnote 1, Section 3.6)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not release participant interaction data, screen recordings, survey responses, or telemetry data. Only analysis scripts are provided, not the underlying data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided in the paper. The GitHub repository link is given but the paper itself does not describe environment setup for reproducing the analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While analysis scripts are released, there are no step-by-step reproduction instructions in the paper. The paper does not provide a README or instructions for replicating the analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors are reported for main results (e.g., task correctness: 'SE = 10%' and 'SE = 11%'; user effort: 'SE = 6.22' and 'SE = 2.8' in Section 4.1). Error bars are shown in Figure 4."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Statistical significance tests are reported: p-values for task correctness (p = 0.02), user effort (p = 0.01), and Wilcoxon Signed-Rank Tests for Likert responses (e.g., C1: p = 0.0013, C2: p = 0.0006). Section 3.6 describes the analysis approach."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context: '35% increase in task correctness' (from 25% to 60%), 'about 50% the user effort' (25.1 min vs 12.5 min) in Section 4.1. Mean and SE provided for both conditions."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided. The paper recruited 20 participants without explaining why this number was chosen or acknowledging it may be insufficient for some analyses."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors are reported for task correctness and user effort metrics (Section 4.1), and error bars are shown in Figure 4. Breakdown analyses in Figures 8 and 9 also show error bars."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study directly compares GitHub Copilot (copilot baseline) against OpenHands (agent). This is the core study design—a within-participant comparison of two tools (Section 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GitHub Copilot is a current, widely-used tool, and OpenHands is described as a 'state-of-the-art, open-source coding agent' (Section 3.2). Both represent contemporary tools as of April-May 2025."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a controlled user study comparing two complete tools, not a system with separable components. An ablation study is not structurally applicable."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: task correctness rate, user effort (time), six Likert-scale user experience comparisons (C1-C6), and qualitative analysis of interaction trajectories (Section 3.5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "This is a human user study. Participants directly evaluated both tools through task performance and post-study surveys. Task correctness was evaluated based on submitted code (Section 3.5)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a user study, not a machine learning evaluation. The concept of held-out test sets does not apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides breakdowns by task type (data analysis, fix bug, add feature), programming experience, copilot usage frequency, and ordering effect (Figures 8 and 9)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where agents underperform: bug-fixing tasks are hardest (Section 4.1), users report poor understanding of agent outputs (C6), agents being overly proactive (Section 5.1), and multiple user experience limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: agents were not preferred for output understanding (C6, 55% preferred copilot), no significant difference on satisfaction (C5, p = 0.26), flow (C4, p = 0.39), or speed (C3, p = 0.09). Participants would still prefer GitHub Copilot overall (F2, p = 0.04)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims agents 'can assist developers in ways that surpass copilots' (supported by 35% task correctness increase), 'reduce the effort required' (supported by ~50% reduction in user effort), and 'challenges remain' (supported by C5, C6, and discussion). All claims are supported in Sections 4.1-4.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The study uses a within-participant controlled design with randomized ordering (Section 3.4), which is adequate for causal claims about tool effects on productivity. Participants solve same-type tasks with both tools. Causal language like 'agents tend to enable users to complete more tasks' is justified by the controlled comparison."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 (Limitations) explicitly bounds generalization: results are for one copilot and one agent, student participants, novice agent users, Python tasks, and a specific point in time (April-May 2025). The paper notes findings 'may not generalize to developers who are working on lower-resource programming languages.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses alternative explanations: uncontrolled LLM choice in copilot phase, novelty effects from first-time agent use, student population bias, and task scope limitations. Appendix D analyzes ordering effects. Section 5.2 compares findings to prior work."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states OpenHands was 'powered by Claude Sonnet 3.7' and that Copilot users could choose from 'GPT-4o, Claude Sonnet 3.7, Gemini 2.5 Pro, and o3-mini' (Section 3.2). These are marketing names without snapshot dates or API version identifiers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates third-party tools (GitHub Copilot and OpenHands) as black boxes. The authors did not design prompts—participants wrote their own messages to the tools. The system prompts of these tools are not under the authors' control."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported for either GitHub Copilot or OpenHands. The OpenHands versions (0.35-0.38) are stated in Appendix A.2, but inference parameters are not."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party tools (GitHub Copilot and OpenHands) as black boxes. While OpenHands's tools are briefly listed (bash shell, Jupyter, browser, file tool in Section 3.2), the authors cannot be expected to fully describe internal scaffolding they do not control."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes data collection methods. Section 3.6 explains the analysis approach: how user effort is computed from event streams, how correctness is evaluated, and how Likert responses are analyzed using Wilcoxon Signed-Rank Tests."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 is a dedicated Limitations section spanning approximately two paragraphs with substantive discussion of multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses specific threats: single instantiation of each tool type, uncontrolled LLM choice in copilot phase, student participant pool, limited number of tasks, 40-minute time constraint not representative of real workflows, task selection bias toward popular languages, and temporal limitation (April-May 2025)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states boundaries: 'our findings are primarily representative of those who are novice coding agent users,' 'findings on the effect of agents on productivity may not generalize to developers who are working on lower-resource programming languages,' and 'our study findings are only representative of the point in time when the study was completed' (Section 5.3)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (interaction logs, survey responses, screen recordings, code submissions) is not released. Only analysis scripts are provided at the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes four data collection methods: user trajectories (screen recordings for copilot, event streams for agent), final code snapshots, Likert responses, and qualitative responses. Appendix C provides full study instructions."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states participants were 'recruited via university mailing lists' with inclusion criteria: 'must have access to and use GitHub Copilot regularly and have experience programming in Python.' Demographics are reported (experience levels, Python proficiency, AI tool usage)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 documents how user effort was computed from event streams (time between agent actions and user messages), how task correctness was evaluated, and how Likert data was analyzed. The pipeline from data collection to analysis is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The Acknowledgements section thanks participants and colleagues but does not mention any funding source, grants, or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Carnegie Mellon University and All Hands AI. All Hands AI is the company behind OpenHands, which is one of the tools being evaluated. This affiliation is visible in the author list."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. However, two authors (Valerie Chen and Graham Neubig) are affiliated with All Hands AI, the maker of OpenHands, which is one of the tools being evaluated. This creates a clear conflict of interest that is not explicitly acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosures statement is present in the paper. Given the All Hands AI affiliation of two authors evaluating OpenHands, this is a notable omission."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a user study evaluating developer interactions with tools, not a benchmark evaluation of model capabilities. Contamination is not the relevant concern here."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a user study, not a benchmark evaluation of pre-trained model capabilities. Train/test overlap is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a user study, not a benchmark evaluation. While tasks were sourced from GAIA and SWE-Bench, the evaluation is of human-AI collaboration, not model capability on benchmark questions."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration link or mention of pre-registration is found in the paper."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval is found in the paper, despite collecting data from 20 human participants including screen recordings."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 reports: 60% had 3-5 years of professional programming experience, 25% had 0-2 years, 15% had 6-10 years. 95% rated themselves intermediate to advanced in Python. All used GitHub Copilot at least weekly."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states inclusion criteria: 'must have access to and use GitHub Copilot regularly and have experience programming in Python.' Participants were students recruited via university mailing lists."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 describes randomization: 'We randomize the order in which participants interact with either [tool]' and 'We use a between-subjects set-up to randomize task types.' Figure 3 illustrates the counterbalanced design."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "Blinding is not described. Participants knew which tool they were using (GitHub Copilot vs OpenHands), which is inherent to the study design. However, evaluator blinding for task correctness grading is not mentioned."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 20 participants were recruited but does not report whether all 20 completed both phases, nor any dropout or exclusion of participant data."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs or token usage are reported for either GitHub Copilot or OpenHands, despite both making substantial LLM API calls during the study."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report API spend, hosting costs for OpenHands, or other compute requirements."
       }
     }
   },
   "claims": [
     {
       "claim": "Agents improve task correctness rate by 35% compared to copilots (25% with copilot vs 60% with agent).",
       "evidence": "Section 4.1, Figure 4: 'we observe a 35% increase in task correctness when users have access to agents as compared to copilots (μ = 25%, SE = 10% compared to μ = 60%, SE = 11% respectively). We find that these productivity improvements are significant (p = 0.02).'",
       "supported": "moderate"
     },
     {
       "claim": "Agents reduce user effort to about 50% of copilot effort (25.1 min vs 12.5 min).",
       "evidence": "Section 4.1: 'We find a significant difference in user effort between the time spent using copilots and agents (p = 0.01), where the former took 25.1 minutes of a user's time on average (SE = 6.22) and the latter only took 12.5 minutes of a user's time (SE = 2.8).'",
       "supported": "moderate"
     },
     {
       "claim": "Agents significantly reduce cognitive load compared to copilots.",
       "evidence": "Section 4.2, Table 2: C2 cognitive load comparison with p = 0.0006, 75% agree/strongly agree, median = 4.0, mean = 3.95.",
       "supported": "moderate"
     },
     {
       "claim": "Agents enable users to accomplish new tasks they could not with copilots.",
       "evidence": "Section 4.2, Table 2: C1 with p = 0.0013, 70% agree/strongly agree, median = 5.0, mean = 4.25. 'Some tasks, such as both of the data analysis problems, were only completed by participants when they had access to agents.'",
       "supported": "moderate"
     },
     {
       "claim": "Users have better understanding of copilot outputs than agent outputs.",
       "evidence": "Section 4.2, Table 2: C6 with 55% disagreeing (preferring copilot), median = 2.0, mean = 2.45, p = 0.07 (not significant at alpha = 0.05).",
       "supported": "weak"
     },
     {
       "claim": "Despite higher productivity with agents, participants would still prefer to use GitHub Copilot over OpenHands.",
       "evidence": "Section 5.1, Table 3: F2 'Moving forward, I would continue to use OpenHands over Github Copilot' with 60% disagreeing, p = 0.04.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["rct", "qualitative"],
   "key_findings": "In a within-participant controlled study with 20 developers, coding agents (OpenHands) improved task correctness by 35% over copilots (GitHub Copilot) and reduced user effort by approximately 50%. However, agents were only significantly preferred on 2 of 6 user experience dimensions (cognitive load and ability to accomplish new tasks). Participants still preferred copilots overall for future use, citing better output understanding and control. The study identifies three design desiderata for coding agents: transparency, balanced proactivity, and effective leverage of human effort.",
   "red_flags": [
     {
       "flag": "Conflict of interest: authors affiliated with evaluated tool",
       "detail": "Two authors (Valerie Chen and Graham Neubig) are affiliated with All Hands AI, the company behind OpenHands, which is one of the two tools being evaluated. This conflict is not explicitly acknowledged in the paper, and there is no competing interests statement."
     },
     {
       "flag": "Small sample size without justification",
       "detail": "The study uses only 20 student participants with no power analysis or sample size justification. With N=20 and a within-participant design, subgroup analyses (by task type, experience, etc.) have very small cell sizes, limiting the reliability of breakdown analyses in Appendix D."
     },
     {
       "flag": "No IRB/ethics approval mentioned",
       "detail": "The study collects screen recordings, interaction logs, and survey data from 20 human participants but does not mention IRB or ethics board approval."
     },
     {
       "flag": "Uncontrolled LLM choice in copilot condition",
       "detail": "Participants could choose any LLM in the GitHub Copilot chat panel (GPT-4o, Claude Sonnet 3.7, Gemini 2.5 Pro, o3-mini), while OpenHands was fixed to Claude Sonnet 3.7. This introduces a confound—differences may be due to model choice rather than tool type."
     },
     {
       "flag": "User effort measured differently across conditions",
       "detail": "User effort for copilots is total time (start to end), while for agents it is only the time spent writing instructions. This asymmetric measurement makes direct comparison problematic, as acknowledged in the paper (total agent time including autonomous work was 27.9 min, comparable to copilot's 25.1 min)."
     },
     {
       "flag": "Novice agent users vs experienced copilot users",
       "detail": "All participants were experienced copilot users but had no prior agent experience. Novelty effects or learning curves could explain some results. The paper acknowledges this but the confound limits interpretation."
     }
   ],
   "cited_papers": [
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "relevance": "Describes the coding agent platform evaluated in this study; relevant to agentic AI software engineering."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "The benchmark from which bug-fixing tasks were sourced; central to evaluating coding agents."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Prior controlled study of GitHub Copilot's productivity impact; key baseline for comparison."
     },
     {
       "title": "Reading Between the Lines: Modeling User Behavior and Costs in AI-Assisted Programming",
       "authors": ["Hussein Mozannar", "Gagan Bansal", "Adam Fourney", "Eric Horvitz"],
       "year": 2024,
       "relevance": "Developed CUPS taxonomy for programmer-AI interactions; directly relevant to understanding developer workflows with AI tools."
     },
     {
       "title": "The RealHumanEval: Evaluating Large Language Models' Abilities to Support Programmers",
       "authors": ["Hussein Mozannar", "Valerie Chen", "Mohammed Alsobay"],
       "relevance": "Demonstrated that benchmark performance may not correlate with downstream user utility for coding assistants."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier", "Thomas Wolf"],
       "year": 2023,
       "relevance": "Source of data analysis tasks used in the study; evaluates AI agents as general-purpose assistants."
     },
     {
       "title": "Grounded Copilot: How Programmers Interact with Code-Generating Models",
       "authors": ["Shraddha Barke", "Michael B. James", "Nadia Polikarpova"],
       "year": 2023,
       "relevance": "Introduced acceleration vs exploration framework for understanding developer-AI interactions."
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "Contemporary study measuring AI productivity impact on developers; directly comparable methodology."
     },
     {
       "title": "The Effects of Generative AI on High-Skilled Work: Evidence from Three Field Experiments with Software Developers",
       "authors": ["Zheyuan Kevin Cui", "Mert Demirer", "Sonia Jaffe"],
       "year": 2025,
       "relevance": "Field experiments on AI's effect on developer productivity; relevant methodological comparison."
     },
     {
       "title": "Vibe Coding: Programming Through Conversation with Artificial Intelligence",
       "authors": ["Advait Sarkar", "Ian Drosos"],
       "year": 2025,
       "arxiv_id": "2506.23253",
       "relevance": "Defines the vibe-coding paradigm discussed in relation to agentic workflows."
     },
     {
       "title": "The Rise of AI Teammates in Software Engineering (SE) 3.0: How Autonomous Coding Agents are Reshaping Software Engineering",
       "authors": ["Hao Li", "Haoxiang Zhang", "Ahmed E. Hassan"],
       "year": 2025,
       "arxiv_id": "2507.15003",
       "relevance": "Tracks adoption of coding agents on GitHub; complementary to this study's controlled evaluation."
     },
     {
       "title": "Collaborative Gym: A Framework for Enabling and Evaluating Human-Agent Collaboration",
       "authors": ["Yijia Shao", "Vinay Samuel", "Yucheng Jiang"],
       "year": 2024,
       "arxiv_id": "2412.15701",
       "relevance": "Framework for evaluating human-agent collaboration, addressing the gap in interactive agent evaluation."
     },
     {
       "title": "Interactive Agents to Overcome Ambiguity in Software Engineering",
       "authors": ["Sanidhya Vijayvargiya", "Xuhui Zhou", "Akhila Yerukola"],
       "year": 2025,
       "arxiv_id": "2502.13069",
       "relevance": "Addresses interactive agent evaluation with synthetic users for software engineering tasks."
     }
   ]
 }