ai-research-survey

scan.json (25833B)

---

 {
   "paper": {
     "title": "Can ChatGPT Support Developers? An Empirical Evaluation of Large Language Models for Code Generation",
     "authors": ["Kailun Jin", "Chung-Yu Wang", "Hung Viet Pham", "Hadi Hemmati"],
     "year": 2024,
     "venue": "MSR '24 (21st International Conference on Mining Software Repositories)",
     "arxiv_id": "2402.11702",
     "doi": "10.1145/3643991.3645074"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or archive reference is provided in the paper. The authors mention a 'publicly available dataset' in their contributions but do not provide a link to their labeled data or analysis code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The authors use the publicly available DevGPT dataset (Xiao et al. 2024) and state as a contribution 'A publicly available dataset of developers and ChatGPT interactions, labelled with the prompt types and the final use cases of the generated code.' The underlying DevGPT dataset is public, though no explicit link is given for the labeled overlay."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or software version details are provided. The study is primarily manual analysis, but there is automated data cleaning that would benefit from reproducibility details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the study design is described at a high level in Section 2, there are no scripts, commands, or detailed procedures that would allow exact replication of the automated filtering or manual labeling process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Results are presented as raw counts and percentages (e.g., '65.3%', '32.8%') without any uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims such as 'conversations in the Commit context lasted on average only 2.4 rounds while conversations in the Code file context lasted on average significantly longer at 10.4 rounds' use the word 'significantly' without any statistical test backing it."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes are reported. The paper presents raw percentages and averages but no measures like Cohen's d or relative effect sizes with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample sizes are reported (2,299 CG conversations, 138 merged PRs) but never justified. No power analysis or discussion of whether 138 merged PRs is sufficient for generalizable conclusions about LLM code usage."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "While box plots are mentioned for RQ1.2 (Figure reference is broken as '??'), the paper does not report standard deviations, IQR values, or other explicit variance measures in the text or tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is an empirical observational/mining study characterizing developer-ChatGPT interactions, not a system evaluation. There is no system to compare against baselines."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is not a system evaluation with baselines. It is a descriptive empirical study."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a mining/observational study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a descriptive study analyzing conversation patterns and code usage, not a system evaluation with performance metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not evaluate a system's outputs. The manual labeling is part of the data collection methodology, not an evaluation of a system."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model is being trained or evaluated on a test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by GitHub category (Pull Request, Issue, Discussion, Commit, Code File) in Table 1, by prompt category in Table 2, and by code usage category in Figure 1 (Exact Match, Modified Code, Document, Supplementary Info)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where generated code was not used (32.8% 'Supplementary Info' — code considered unhelpful) and gives a specific example: 'a commit conversation indicates that the generated code is considered unhelpful, causing a notable slowdown instead of enhancing code execution speed.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 32.8% of generated code was not directly used and that 'the current practice of using LLM-generated code is typically limited to either demonstrating high-level concepts or providing examples in documentation, rather than to be used as production-ready code.' This is a negative finding about LLM code utility."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'the current practice of using LLM-generated code is typically limited to either demonstrating high-level concepts or providing examples in documentation, rather than to be used as production-ready code.' This is supported by Figure 1 showing only 16.8% Exact Match and 26% Modified Code, with the majority being Document (24.4%) or Supplementary Info (32.8%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-sounding claims without adequate justification. For example: 'when developers provide specific code snippets directly to ChatGPT for improvement (in the Commit context), the desirable outcomes are likely achieved quicker with fewer clarifications.' This is a causal inference from observational data without controlling for confounds. The speculation about transformer 'memory' causing developers to avoid 'Request another generation' (Section 3.1) is presented as explanatory without evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title asks 'Can ChatGPT Support Developers?' broadly, and the conclusion states 'Much improvement is needed before LLMs become an integral part of modern software development.' However, the data comes only from publicly shared ChatGPT links on GitHub (a self-selected, survivorship-biased sample), covering only GPT-3.5 conversations from mid-2023. These significant scope limitations are not reflected in the title or conclusion."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 (Discussion) explicitly discusses survivorship bias and filtering bias: 'developers are less likely to include ChatGPT URLs in commit messages if the generated code's quality is too low. This may skew the dataset towards resolved discussions.' These are specific alternative explanations for the observed patterns."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'ChatGPT' and 'GPT-3.5' without specifying exact model versions or snapshot dates. No API version or model identifier (e.g., gpt-3.5-turbo-0613) is stated. Since the data comes from DevGPT (shared conversations), the exact model versions used by developers are unknown and this is not acknowledged."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the authors did not prompt an LLM themselves. They analyzed existing developer conversations with ChatGPT from the DevGPT dataset. The prompts they study are the developers' prompts, which are part of the dataset."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the authors did not run any LLM inference. They analyzed an existing dataset of conversations."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no agentic scaffolding is used. This is a mining study analyzing existing conversations."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the automated data cleaning steps: filtering out Hacker News conversations, extracting code-generation conversations by checking for 'ListofCode' attributes, and manually removing 344 irrelevant conversation rounds from 2,299 conversations. Section 2.3 describes filtering 189 PR conversations down to 138 merged PRs by excluding 51 non-merged ones."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 (Discussion) serves as a limitations section, discussing survivorship bias, filtering bias, and dataset limitations. While not titled 'Limitations' explicitly, it provides substantive discussion of threats to validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 discusses specific threats: survivorship bias in the DevGPT dataset ('constructed from retained conversations, potentially overlooking a broader spectrum of developer interactions') and filtering bias ('developers are less likely to include ChatGPT URLs in commit messages if the generated code's quality is too low'). These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It acknowledges 'dataset limitations based on empirical experience' but does not enumerate specific out-of-scope claims. The generalization from shared GitHub ChatGPT links to 'developers' broadly is never explicitly bounded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying DevGPT dataset is publicly available (Xiao et al. 2024, reference [24]), which forms the raw data for this study. The conversations can be independently accessed and verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes how DevGPT was collected: 'searching with the keyword \"https://chat.openai.com/share/\" in the GitHub GraphQL API to identify mentions of shared links sourced from software development artifacts.' The dataset covers six snapshots from July 24th to August 31st, 2023, containing 17,913 prompts and 11,751 code snippets."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the crowd-sourcing labeling, the paper says only 'we distributed the 2,299 valid data entries to volunteers whom we recruited from our local software engineering community.' No details are given on how many volunteers participated, what their qualifications were, how they were recruited, or whether this introduces selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: DevGPT dataset → filter out Hacker News → parse JSON for code generation conversations → remove 344 irrelevant rounds → 2,299 CG conversations. For RQ2: 189 PR conversations → exclude 51 non-merged → 138 merged PRs. Each stage with counts is described in Sections 2.2 and 2.3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Acknowledgement) states: 'This work was partly supported by NSERC and Alberta Innovates.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as affiliated with York University, Toronto, Canada. No conflict with the evaluated product (ChatGPT/OpenAI) — the authors are independent academic researchers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSERC (Natural Sciences and Engineering Research Council of Canada) and Alberta Innovates are government funding agencies with no financial interest in ChatGPT's code generation performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. While the authors appear to be independent academics, the absence of an explicit declaration is noted."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a mining study analyzing developer conversations. It does not evaluate a pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation of a model is performed. The study analyzes existing conversations."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a repository mining study. The crowd-sourced labelers are annotators, not study participants. No human subjects experiment is conducted."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a mining study of publicly shared conversations. No human subjects research is conducted."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants study. The developers whose conversations are analyzed are not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no experimental study with human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no experimental study with human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study does not propose or run a method that incurs inference cost. It is a mining study analyzing existing conversations."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a mining/manual analysis study, not a compute-intensive method."
       }
     }
   },
   "claims": [
     {
       "claim": "65.3% of ChatGPT conversations on GitHub are related to code generation.",
       "evidence": "Table 1 shows 2,299 CG-related conversations out of 3,523 total (Section 3.1).",
       "supported": "strong"
     },
     {
       "claim": "Only about half (52.3%) of code file conversations are directly associated with code generation, the lowest among all GitHub categories.",
       "evidence": "Table 1 shows 1,052 CG out of 2,010 Code File conversations (Section 3.1, RQ1.1).",
       "supported": "strong"
     },
     {
       "claim": "Commit-related conversations predominantly involve code generation (98.5%) and require fewer prompt-response rounds (average 2.4 rounds).",
       "evidence": "Table 1 shows 660/670 Commit conversations are CG. Section 3.1 RQ1.2 states average 2.4 rounds for Commits.",
       "supported": "moderate"
     },
     {
       "claim": "32.8% of generated code is not used (Supplementary Info), indicating the current practice is limited to high-level concepts or documentation rather than production-ready code.",
       "evidence": "Figure 1 shows usage distribution: 32.8% Supplementary Info, 24.4% Document, 26% Modified, 16.8% Exact Match (Section 3.2).",
       "supported": "moderate"
     },
     {
       "claim": "Much improvement is needed before LLMs become an integral part of modern software development.",
       "evidence": "Based on the finding that only 16.8% of generated code is used as-is (Exact Match) and 32.8% is not used at all (Section 3.2, Conclusion Section 5).",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["observational", "qualitative"],
   "key_findings": "This MSR 2024 paper analyzes 3,523 developer-ChatGPT conversations from the DevGPT dataset. It finds that 65.3% of conversations involve code generation, with Commit-related interactions focused on code improvement (98.5% CG-related, average 2.4 rounds) while Code File conversations are longer (10.4 rounds) and often involve clarification. Analysis of 138 merged pull requests shows that only 16.8% of generated code is used as-is, 26% is heavily modified, 24.4% appears in documentation, and 32.8% is not used at all, suggesting LLM-generated code is primarily used for conceptual demonstrations rather than production code.",
   "red_flags": [
     {
       "flag": "Survivorship bias in dataset",
       "detail": "The DevGPT dataset only captures conversations that developers chose to share publicly via GitHub. Conversations where ChatGPT was unhelpful are less likely to be shared, systematically biasing the sample toward more successful interactions. The authors acknowledge this but do not quantify or mitigate it."
     },
     {
       "flag": "No statistical tests despite comparative claims",
       "detail": "The paper uses the word 'significantly' when comparing conversation lengths across categories (2.4 vs 10.4 rounds) without any statistical test. Multiple comparative claims are made based on raw percentage differences without significance testing."
     },
     {
       "flag": "Small sample for RQ2",
       "detail": "RQ2 conclusions about how generated code is used are based on only 138 merged pull requests. This is a small sample from which to draw general conclusions about LLM code utility in software development, and no justification is provided for this sample size."
     },
     {
       "flag": "Overclaimed scope",
       "detail": "The title asks 'Can ChatGPT Support Developers?' and the conclusion generalizes to 'LLMs' and 'modern software development,' but the data covers only GPT-3.5 conversations from a 5-week window in mid-2023, captured through a specific sharing mechanism on GitHub. The generalization far exceeds the evidence."
     },
     {
       "flag": "Broken figure references",
       "detail": "The paper text contains 'Figure ??' references (Section 3.1, RQ1.2), suggesting a LaTeX compilation issue. Key results about conversation length distributions reference figures that cannot be examined."
     },
     {
       "flag": "Crowd-sourced labeling with minimal inter-rater details",
       "detail": "The crowd-sourcing for RQ1.3 uses majority voting among 3 annotators from a 'local software engineering community,' but no inter-rater reliability metrics (e.g., Fleiss' kappa) are reported. The number of labelers, their qualifications, and disagreement rates are not provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational Codex paper on evaluating LLM code generation capabilities, directly relevant to code generation evaluation methodology."
     },
     {
       "title": "Grounded copilot: How programmers interact with code-generating models",
       "authors": ["Shraddha Barke", "Michael B James", "Nadia Polikarpova"],
       "year": 2023,
       "relevance": "Studies developer interactions with code-generating models, relevant to understanding AI-assisted programming workflows."
     },
     {
       "title": "Taking Flight with Copilot: Early Insights and Opportunities of AI-Powered Pair-Programming Tools",
       "authors": ["Christian Bird", "Denae Ford", "Thomas Zimmermann"],
       "year": 2023,
       "doi": "10.1145/3582083",
       "relevance": "Early empirical study of Copilot adoption and developer experience, relevant to AI programming tool evaluation."
     },
     {
       "title": "Investigating Code Generation Performance of Chat-GPT with Crowdsourcing Social Data",
       "authors": ["Yunhe Feng"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT code generation using crowdsourcing, directly relevant to LLM code generation assessment methodology."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Rigorous evaluation of LLM-generated code correctness, central to assessing code generation quality."
     },
     {
       "title": "Understanding the Usability of AI Programming Assistants",
       "authors": ["Jenny T Liang", "Chenyang Yang", "Brad A Myers"],
       "year": 2023,
       "arxiv_id": "2303.17125",
       "relevance": "Studies usability of AI programming tools, relevant to developer productivity and tool design."
     },
     {
       "title": "Prompt Engineering or Fine Tuning: An Empirical Assessment of Large Language Models in Automated Software Engineering Tasks",
       "authors": ["Jiho Shin", "Clark Tang", "Tahmineh Mohati"],
       "year": 2023,
       "arxiv_id": "2310.10508",
       "relevance": "Empirical comparison of prompt engineering vs fine-tuning for SE tasks; source of the prompt categorization taxonomy used in this paper."
     },
     {
       "title": "Security implications of large language model code assistants: A user study",
       "authors": ["Gustavo Sandoval", "Hammond Pearce"],
       "year": 2022,
       "arxiv_id": "2208.09727",
       "relevance": "User study on security implications of LLM code assistants, relevant to AI code safety and quality."
     },
     {
       "title": "DevGPT: Studying Developer-ChatGPT Conversations",
       "authors": ["Tao Xiao", "Christoph Treude", "Hideaki Hata", "Kenichi Matsumoto"],
       "year": 2024,
       "relevance": "Source dataset for this study; characterizes developer-ChatGPT conversations on GitHub."
     },
     {
       "title": "A systematic evaluation of large language models of code",
       "authors": ["Frank F Xu", "Uri Alon", "Graham Neubig", "Vincent Josua Hellendoorn"],
       "year": 2022,
       "relevance": "Systematic evaluation of code LLMs, directly relevant to code generation evaluation methodology."
     },
     {
       "title": "A study on robustness and reliability of large language model code generation",
       "authors": ["Li Zhong", "Zilong Wang"],
       "year": 2023,
       "arxiv_id": "2308.10335",
       "relevance": "Studies robustness and reliability of LLM code generation, relevant to code quality assessment."
     },
     {
       "title": "Software testing with large language model: Survey, landscape, and vision",
       "authors": ["Junjie Wang", "Yuchao Huang"],
       "year": 2023,
       "arxiv_id": "2307.07221",
       "relevance": "Survey of LLMs for software testing, relevant to the broader landscape of AI-assisted software engineering."
     }
   ]
 }