ai-research-survey

scan.json (24280B)

---

 {
   "paper": {
     "title": "Does Co-Development with AI Assistants Lead to More Maintainable Code? A Registered Report",
     "authors": [
       "Markus Borg",
       "Dave Hewett",
       "Donald Graham",
       "Noric Couderc",
       "Emma Söderberg",
       "Luke Church",
       "Dave Farley"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2408.10758",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is available on Zenodo (https://zenodo.org/records/12827815), referenced in Section VII and [17]. The code base and tasks are shared as PDF documents on Zenodo, though the actual code is not in a public git repository to prevent leakage to AI training data."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "This is a registered report describing a planned study. No experimental data has been collected yet, so no data can be released. The replication package contains the study materials (task descriptions, code base) but not experimental results data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Java/Spring Boot and references the snapcode.review platform, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the study execution plan in detail (Section VIII) and provides a Zenodo replication package [17] with task instructions and the code base. As a registered report, the instructions are for replicating the study protocol rather than computational results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII states: 'We will perform standard t-tests and report effect sizes and confidence intervals in accordance with the empirical standard.' While this is a planned analysis (no results yet), the pre-registered commitment to report CIs is documented."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section V specifies t-tests at α = 0.05 for normally distributed variables and rank-sum tests for ordinal (Likert) data. Four null hypotheses (H01–H04) are formally stated."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V states: 'We will complement the p-values by reporting effect sizes.' Section VIII reiterates: 'report effect sizes and confidence intervals.' The power analysis uses d=0.5 (medium effect size). Pre-registered commitment to report effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section VI includes a formal power analysis: 'We used G*Power (v.3.1.9.7) to run a power analysis for a two-tailed t-test... Based on our expectation of medium effect sizes (d=0.5), α = 0.05, and a power of 0.80, the result indicates that we need at least 64 participants per group.' Total target: 256."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results are reported in this registered report, and the pre-registration does not explicitly commit to reporting variance or standard deviations across observations, though this would be expected as part of the planned t-tests."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The RCT design includes a control group (!AI-devs) whose code is evolved alongside treatment code (AI-devs). This is a proper baseline comparison built into the experimental design (Section III, Figure 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is the contemporary practice of coding without AI assistants vs. with AI assistants. This is inherently contemporary since it compares current development practices."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a controlled experiment comparing two conditions (AI-assisted vs. not), not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four dependent variables are measured: completion time, Code Health (CH), test coverage (TC), and perceived productivity (PP) using the SPACE framework (Section IV, Figure 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Perceived productivity is measured via human self-assessment using a Likert-scale exit questionnaire (Table II, Q2-6) based on the SPACE framework. The entire study centers on human participants performing coding tasks."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a controlled experiment with human participants, not a machine learning evaluation. The concept of held-out test sets does not apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No results are reported yet (registered report). The pre-registration mentions secondary analysis controlling for confounders (Section VIII) but does not explicitly commit to per-category breakdowns (e.g., by developer experience level)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No results are reported yet. As a registered report, no failure cases can be discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results are reported yet. The commitment to report all results (positive or negative) is implicit in the registered report format but not explicitly stated."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract makes no empirical claims about results — it describes the planned method. All statements in the abstract accurately reflect the study design described in the paper. The abstract correctly states 'We will conduct a two-phased controlled experiment.'"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper proposes to make causal claims via an RCT design. The causal graph is explicitly modeled using DAGitty (Section IV, Figure 3), confounding variables are identified (Table IV), and the Phase 2 RCT with randomized assignment provides adequate basis for causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VII explicitly states: 'the conclusions drawn from studying this specific task cannot be generalized to all possible development scenarios' and acknowledges threats from 'our choice of a single programming language and the complexity of the task.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IX (Risk Analysis and Threats to Validity) discusses multiple alternative explanations: participant imbalance, different interpretations of 'AI assistant', non-compliance with instructions, developer experience as a confounder. The DAGitty causal graph (Figure 3) explicitly models confounders."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any specific AI model. It studies the general phenomenon of AI-assisted development; participants choose their own AI assistant tool. This is a study of developer behavior, not model capability."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as part of its methodology. Participants use their own AI assistants with their own prompts. The study is about the effect on code maintainability, not about specific prompts."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLM hyperparameters are part of the study design. Participants choose their own tools and settings. The statistical analysis hyperparameters (α = 0.05, d = 0.5) are reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Participants use off-the-shelf AI assistants (e.g., GitHub Copilot) as black boxes within their IDEs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII (Data cleaning) specifies: exclude participants whose solutions fail acceptance tests, remove those who didn't adhere to instructions (per exit questionnaire), and remove outlier submissions indicating gaming. Pre-registered data cleaning criteria are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section IX is titled 'Risk Analysis and Threats to Validity' and provides substantive discussion of threats to the study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section IX discusses specific threats: failing to attract enough participants, imbalance between AI-devs and !AI-devs, participants interpreting 'AI assistant' differently, non-compliance with instructions in remote settings. Section VII also discusses external validity threats from single programming language and task complexity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VII explicitly states: 'the conclusions drawn from studying this specific task cannot be generalized to all possible development scenarios.' It further notes 'The main threats to external validity stem from our choice of a single programming language and the complexity of the task.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No data has been collected yet (registered report). The replication package on Zenodo contains study materials but no experimental data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII describes data collection in detail: the snapcode.review platform administers tasks, timestamps are collected from repository access to submission, exit questionnaires gather self-reported data, and code quality is measured via CodeScene's Code Health metric."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section VI describes recruitment: 'We will recruit participants through i) social media advertisements on platforms such as YouTube, LinkedIn, and X and ii) using our personal networks.' Pre-screening questionnaire (Table I) is fully documented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: recruitment → pre-screening questionnaire → stratified random assignment → Phase 1 task → Phase 2 RCT → exit questionnaire → data cleaning (acceptance test filtering, compliance filtering, outlier removal) → Bayesian and frequentist analysis."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgment section is present in the paper. The involvement of CodeScene (first author's affiliation) and Equal Experts is mentioned but formal funding disclosure is absent."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Markus Borg at CodeScene and Lund University, Dave Hewett and Donald Graham at Equal Experts, others at Lund University, Dave Farley at Continuous Delivery."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure is provided. The first author is affiliated with CodeScene, whose Code Health metric is used as a dependent variable. Equal Experts provides the snapcode.review platform. The independence of these organizations from the study outcome is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. The first author's dual affiliation with CodeScene (a commercial company whose metric is used in the study) represents a potential conflict that is not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model's capability on any benchmark. It evaluates the effect of AI assistants on human developer code quality."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a model benchmark evaluation. However, the paper does address a related contamination concern: Section VII notes the code base is not hosted publicly to prevent 'leakage to AI assistants' training data' [19]."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a model benchmark evaluation. The paper studies human developer behavior, not model capability."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": true,
         "justification": "This IS a registered report — the entire paper is a pre-registration of the study design submitted for peer review before data collection. This is the gold standard for pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned. Section VI states the study 'adheres to the essential attributes of the ACM SigSoft Empirical Standard Ethics (Studies with Human Participants)' but no institutional ethics review is referenced."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "The pre-screening questionnaire (Table I) collects: gender (Q1-1), age (Q1-2), location (Q1-3), occupation type (Q1-4), programming experience (Q1-5), Java proficiency (Q1-6), and AI assistant experience (Q1-7, Q1-8). Demographics will be collected though no data exists yet."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section VI describes criteria: AI-dev cohort requires affirmative Q1-7, agreement with Q1-8a, and positive mean on Q1-8b-i. Q1-6 is used to filter out Java novices. Data cleaning excludes participants failing acceptance tests or not adhering to instructions."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III and VI describe randomization: 'random stratified sampling to split the participants into either Task 1 or Task 2.' Phase 2 uses random assignment without replacement to evolve Task 1 solutions. Stratification ensures equal AI-dev/!AI-dev distribution."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "Blinding is not discussed. Phase 2 participants evolve code from Phase 1 but it is not stated whether they know if the code was AI-assisted or not. This is a notable omission — knowing whether code was AI-generated could affect behavior."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No data has been collected yet, so attrition cannot be reported. The paper discusses potential dropout risks (Section IX) but does not commit to specific attrition reporting (e.g., CONSORT flow diagram)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a controlled experiment studying human developers, not a computational method. Inference cost is not applicable."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a controlled experiment studying human developers. Computational budget is not applicable."
       }
     }
   },
   "claims": [
     {
       "claim": "AI assistants like GitHub Copilot are transforming software engineering with productivity improvements.",
       "evidence": "Cited studies: Peng et al. found 55.8% faster completion with Copilot [3], Ziegler et al. report large positive impact on perceived productivity [4], JetBrains survey shows 77% use ChatGPT and 46% use Copilot [1]. Section I and II.",
       "supported": "moderate"
     },
     {
       "claim": "The study requires 256 participants (64 per group in Phase 2) for adequate statistical power.",
       "evidence": "Section VI reports G*Power analysis: two-tailed t-test, medium effect size d=0.5, α=0.05, power=0.80 yields 64 per group. Total 256 accounts for both phases.",
       "supported": "strong"
     },
     {
       "claim": "A two-phase RCT design can isolate the causal effect of AI-assisted code on maintainability.",
       "evidence": "Section III-IV describe the design with a DAGitty causal graph (Figure 3), stratified random assignment, control for confounders (Table IV), and separation of Phase 1 code creation from Phase 2 evolution.",
       "supported": "strong"
     },
     {
       "claim": "CodeScene's Code Health metric is associated with defect counts and development effort.",
       "evidence": "Section IV cites [15] Tornhill & Borg (2022) and [16] Borg et al. (2024) as supporting evidence for using CH as a maintainability measure.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "rct"
   ],
   "key_findings": "This is a registered report presenting the design for a two-phase controlled experiment on AI assistant impact on code maintainability. No empirical findings are reported. The study proposes an RCT where professional developers evolve Java code previously written with or without AI assistants, measuring completion time, Code Health, test coverage, and perceived productivity. A power analysis targets 256 participants with Bayesian and frequentist analyses planned.",
   "red_flags": [
     {
       "flag": "No empirical results",
       "detail": "This is a registered report describing planned research. No data has been collected and no results are reported. The value lies entirely in the study design, not findings."
     },
     {
       "flag": "Potential conflict of interest with CodeScene",
       "detail": "First author Markus Borg is affiliated with CodeScene, and CodeScene's proprietary Code Health metric is used as one of four dependent variables. No conflict of interest statement is provided, and no alternative code quality metrics are included for triangulation."
     },
     {
       "flag": "No IRB/ethics approval mentioned",
       "detail": "The study involves human participants completing 2-4 hour tasks but no institutional ethics review or IRB approval is mentioned. The paper cites ACM SigSoft ethical standards but this is self-assessed compliance, not independent review."
     },
     {
       "flag": "Blinding not addressed",
       "detail": "Phase 2 participants evolve Phase 1 code, but whether they are blinded to the AI-assistance condition is not discussed. Knowledge of whether code was AI-assisted could influence behavior and perceptions."
     },
     {
       "flag": "Ambitious recruitment target",
       "detail": "The study targets 256 professional developer volunteers for a 2-4 hour unpaid coding task (with only raffle incentives). Section IX acknowledges this as the 'main risk' but the feasibility of reaching this number is uncertain."
     }
   ],
   "cited_papers": [
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["S. Peng", "E. Kalliamvakou", "P. Cihon", "M. Demirer"],
       "year": 2023,
       "relevance": "Seminal RCT showing 55.8% faster task completion with GitHub Copilot; key comparison study for AI-assisted development productivity."
     },
     {
       "title": "Measuring GitHub Copilot's Impact on Productivity",
       "authors": ["A. Ziegler", "E. Kalliamvakou", "X. A. Li", "A. Rice", "D. Rifkin", "S. Simister", "G. Sittampalam", "E. Aftandilian"],
       "year": 2024,
       "relevance": "GitHub's survey-based study of Copilot's perceived productivity impact using the SPACE framework; directly adapted by this paper."
     },
     {
       "title": "GitHub Copilot AI pair Programmer: Asset or Liability?",
       "authors": ["A. Moradi Dakhel", "V. Majdinasab", "A. Nikanjam", "F. Khomh", "M. C. Desmarais", "Z. M. J. Jiang"],
       "year": 2023,
       "relevance": "Evaluation of Copilot's code generation quality showing variable quality and defects; relevant to AI code quality assessment."
     },
     {
       "title": "Do Users Write More Insecure Code with AI Assistants?",
       "authors": ["N. Perry", "M. Srivastava", "D. Kumar", "D. Boneh"],
       "year": 2023,
       "relevance": "User study (n=47) finding AI-assisted participants produced less secure code; highlights quality risks of AI coding tools."
     },
     {
       "title": "Is GitHub's Copilot as Bad as Humans at Introducing Vulnerabilities in Code?",
       "authors": ["O. Asare", "M. Nagappan", "N. Asokan"],
       "year": 2023,
       "relevance": "Empirical study comparing vulnerability introduction rates between Copilot and human developers."
     },
     {
       "title": "How Readable is Model-generated Code? Examining Readability and Visual Inspection of GitHub Copilot",
       "authors": ["N. Al Madi"],
       "year": 2023,
       "relevance": "Controlled experiment (n=21) on code readability of AI-assisted code, finding comparable quality to human pair programming."
     },
     {
       "title": "The Recent Trends of Research on GitHub Copilot: A Systematic Review",
       "authors": ["Z. C. Ani", "Z. A. Hamid", "N. N. Zhamri"],
       "year": 2024,
       "relevance": "Systematic literature review of GitHub Copilot research; maps the landscape of AI assistant studies."
     },
     {
       "title": "The SPACE of Developer Productivity: There's more to it than you think",
       "authors": ["N. Forsgren", "M.-A. Storey", "C. Maddila", "T. Zimmermann", "B. Houck", "J. Butler"],
       "year": 2021,
       "relevance": "Foundational framework for measuring developer productivity used as basis for perceived productivity measurement in AI-assisted development studies."
     },
     {
       "title": "Code Red: The Business Impact of Code Quality - A Quantitative Study of 39 Proprietary Production Codebases",
       "authors": ["A. Tornhill", "M. Borg"],
       "year": 2022,
       "relevance": "Validates CodeScene's Code Health metric against defect counts and development effort; provides construct validity for maintainability measurement."
     }
   ]
 }