ai-research-survey

scan.json (27565B)

---

 {
   "paper": {
     "title": "Analysis of Student-LLM Interaction in a Software Engineering Project",
     "authors": [
       "Agrawal Naman",
       "Ridwan Shariffdeen",
       "Guanlin Wang",
       "Sanka Rasnayaka",
       "Ganesh Neelakanta Iyer"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2502.01273"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, Zenodo archive, or any code release is mentioned in the paper. The analysis scripts, conversation data processing code, and complexity measurement tools are not provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The collected dataset of 730 code snippets, 62 ChatGPT conversations, and student annotations is not released. No download link or data repository is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or library versions are mentioned. The tools used for analysis (Tree-sitter, VADER, complexity metrics) are named but no setup details are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology section describes the study design at a high level but does not include commands, scripts, or detailed procedures to replicate the analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results. The paper presents density plots, histograms, and box plots but without uncertainty quantification. Figures show distributions but point estimates lack CI notation or ± values."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'Copilot generated significantly more outliers than GPT', 'ChatGPT generates responses with lower computational complexity compared to CoPilot') but no statistical significance tests (t-tests, Mann-Whitney U, etc.) are reported. Comparisons are based solely on visual inspection of plots and raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes (Cohen's d, odds ratios, etc.) are reported. The paper provides raw percentages (e.g., '53.6% required minor intervention', '26% requiring no manual modification') but without formal effect size measures or baseline context sufficient for the reader to assess magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of 126 students in 21 groups is stated but no justification for why this is adequate is given. No power analysis is discussed. The 62 ChatGPT conversations and 730 code snippets are described without justifying whether these counts are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, interquartile ranges, or other variance measures are reported in numerical form. Box plots in figures show some distributional information visually, but no numerical variance or spread measures accompany the reported results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares ChatGPT-generated code against Copilot-generated code across complexity metrics, and also compares generated code against repository-integrated code. The comparison between the two LLM tools serves as a mutual baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ChatGPT and GitHub Copilot are contemporary tools that were actively used by students during the study period (2024 academic year). These represent current state-of-the-art LLM code generation tools."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study of student behavior, not a system with components that can be ablated. There is no system design with removable components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The study uses four code complexity metrics: Total Lines of Code (LOC), Cyclomatic Complexity, Maximum Control Flow Graph (CFG) Depth, and Halstead Effort. It also uses Jaccard similarity for code comparison and VADER compound scores for sentiment analysis."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "There is no human evaluation of the quality of the LLM-generated code outputs. Students self-reported intervention levels (0, 1, 2) but there was no systematic expert review or human rating of code quality. The intervention levels are self-reported metadata, not a structured human evaluation of system outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study analyzing student interaction data, not a benchmark evaluation. There is no train/test split concept applicable here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by tool (ChatGPT vs Copilot), by milestone (MS1, MS2, MS3), by team (Table I, Table II), by code type (test vs functionality), and by intervention level. Per-team breakdowns are provided in Tables I and II."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where AI-generated code was removed or refactored (teams 5, 8, 10, 13, 17 showed declining cumulative snippets), and notes that Copilot produced 'highly complex solutions, sometimes exceeding the average values by 40 to 50 times.' Mid-conversation frustration patterns are also discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 5 out of 21 teams did not use any LLMs despite having access, that Copilot occasionally produced unnecessarily complex code, that AI usage declined across milestones, and that sentiment analysis showed mid-conversation frustration. These are negative or mixed findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that students prefer ChatGPT over Copilot (supported by usage data in Table I showing higher ChatGPT adoption by more teams), that ChatGPT generates lower complexity code (supported by Figure 2 and density plots), and that conversational interaction improves code quality (supported by Figure 3 showing decreasing complexity over conversation). However, the claim about 'preference' is based on usage patterns rather than stated preference, which is a stretch."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'conversational-based interaction helps improve the quality of the code generated' and 'ChatGPT's conversational interface enables users to iteratively refine prompts, resulting in more efficient code generation.' These are causal assertions from an observational study without controlling for confounds (e.g., self-selection: students who chose ChatGPT may differ from those who chose Copilot; task complexity differences are not controlled)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'Early adoption of LLMs in software engineering is crucial to remain competitive' and 'the next generation of software engineers must acquire the necessary skills to interact with AI' — these are broad generalizations from a single-course study at one university with one specific project type (SPA). The title 'Analysis of Student-LLM Interaction in a Software Engineering Project' is appropriately scoped, but the conclusions extend well beyond the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the lower complexity of ChatGPT code could be due to different task types being given to ChatGPT vs Copilot, or self-selection effects (students who prefer simpler code might choose ChatGPT). The declining usage across milestones could reflect project phase characteristics rather than student learning. The Threats to Validity section mentions self-reporting bias and VADER limitations but does not address alternative explanations for the core findings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'ChatGPT' and 'GitHub Copilot' without specifying model versions, snapshot dates, or API versions. No GPT model version (e.g., GPT-3.5, GPT-4, specific API version) is stated. The 'paid versions of ChatGPT via both the Chat and Playground interfaces' is mentioned but not which underlying model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides example prompts in Chat Listings 1, 2, and 3, showing actual prompt text used by students. While not all 318 messages are reproduced, representative examples with actual text are provided. However, this is a study of student prompting behavior, not a system that uses fixed prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API settings are reported for either ChatGPT or Copilot. The paper does not mention what settings students used when interacting with the LLMs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates students using ChatGPT and Copilot as third-party tools (black boxes). There is no custom agentic scaffolding built by the authors."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II describes the data collection procedure: students tagged LLM-generated code with generator used, intervention level, and conversation links. The code extraction process follows their prior work [3]. The analysis pipeline for conversations is described: Tree-sitter extraction, Jaccard similarity calculation with LCS threshold of 90%, and the filtering approach for conversation analysis (excluding reused code, omitting zero-similarity conversations)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section IV 'Threats to Validity' is a dedicated section discussing limitations of the study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Threats to Validity section identifies specific threats: (1) voluntary self-reports may include underreporting or selective disclosure, (2) Copilot chat feature was not widely used limiting comparison, (3) VADER misclassifies technical terms as neutral, resulting in many near-zero scores. These are specific to this study rather than generic boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There are no explicit boundaries on generalization — no statement about the results being limited to this specific course, project type (SPA), university (NUS), or student population. The conclusions section makes broad recommendations without qualifying their scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw conversation logs, code snippets, student annotations, and team repository data are not released or made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section II describes the data collection: students tagged LLM-generated code with generator, intervention level, and conversation links at each milestone. Collection was cumulative across three milestones. ChatGPT access was through organizational accounts enabling monitoring. The process follows their prior work [3] with addition of conversation link collection."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The participants are 126 undergraduate students enrolled in a software engineering course at the National University of Singapore. The recruitment is implicit — they are course participants, not separately recruited. The study was conducted within the course with DERC approval. This is a convenience sample of an entire course cohort."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is described: students tag code at each milestone → code snippets extracted with metadata → conversations collected via organizational ChatGPT access → complexity metrics computed (LOC, cyclomatic complexity, CFG depth, Halstead effort) → similarity computed using Tree-sitter + LCS + Jaccard → sentiment analyzed with VADER. However, specific counts of filtered/removed data at each stage are not always provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or grant acknowledgment is mentioned in the paper. The university funded ChatGPT and Copilot access but no explicit funding disclosure section or acknowledgments section is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with School of Computing, National University of Singapore, with email addresses provided. They are evaluating tools (ChatGPT, Copilot) they have no commercial affiliation with."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper mentions the university funded ChatGPT and Copilot access for the study but does not disclose whether there was any external research funding or state that the work was unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study analyzes student interaction patterns with LLMs in an educational context. It does not evaluate model capability on any benchmark. The focus is on how students use the tools, not on testing model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study does not evaluate a pre-trained model on any benchmark. It is an observational study of student-LLM interaction patterns."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation is conducted. The study examines student behavior and code characteristics, not model benchmark performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration link or mention of pre-registration is found in the paper."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section II states: 'In compliance with institutional guidelines, approval for our research was obtained from the Departmental Ethics Review Committee (DERC) before conducting the study.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '126 undergraduate students' and '21 groups' but provides no demographic details such as year of study, prior programming experience, gender distribution, age, or other characteristics. The students are not characterized beyond being undergraduates in a software engineering course."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for participants are stated. The implicit criterion is course enrollment, but no screening or selection criteria are described. Five teams that did not use LLMs are mentioned but it is unclear whether their data was excluded from all analyses."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study, not an experimental study with treatment conditions. Students were not randomly assigned to use ChatGPT or Copilot — they had access to both and chose which to use. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study, not an experimental study. There are no treatment conditions to blind. Students naturally chose which tools to use."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 126 students in 21 groups but does not report whether all students completed the semester, whether any dropped out, or how many students in each team actually engaged with LLMs. Five teams did not use LLMs but attrition within teams is not discussed."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational educational study, not a system or method with inference costs. The paper studies how students use existing tools, not a new method being proposed."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational educational study. The analysis involves standard complexity metric computation and sentiment analysis, not significant computational work requiring budget disclosure."
       }
     }
   },
   "claims": [
     {
       "claim": "Students prefer ChatGPT over Copilot for code generation in software engineering projects.",
       "evidence": "Table I shows cumulative model usage, but ChatGPT total (223 snippets at MS3) is actually lower than Copilot (507 snippets at MS3). More teams used Copilot (6 primarily Copilot vs 5 primarily ChatGPT). The abstract claim of preference is not clearly supported by the usage data.",
       "supported": "weak"
     },
     {
       "claim": "ChatGPT generates responses with lower computational complexity compared to Copilot.",
       "evidence": "Figure 1 (density plots) and Figure 2 (comparison box plots) show that Copilot-generated code has higher Halstead Effort and more outliers across complexity metrics. Section III.B discusses this finding.",
       "supported": "moderate"
     },
     {
       "claim": "Conversational-based interaction helps improve the quality of the code generated compared to auto-generated code.",
       "evidence": "Figure 3 shows average code complexity decreasing across conversation turns. Chat Listing 1 provides a qualitative example. However, no statistical tests compare conversational vs auto-generated code quality, and the comparison conflates tool differences with interaction mode differences.",
       "supported": "weak"
     },
     {
       "claim": "Students demonstrate improved prompting skills over the semester.",
       "evidence": "Chat Listings 2 and 3 show qualitative examples of prompts becoming more detailed and project-specific from MS1 to MS3. Figure 5 shows increasing similarity between generated and integrated code across milestones. However, this is based on qualitative examples and indirect evidence (similarity scores), not systematic measurement of prompt quality.",
       "supported": "moderate"
     },
     {
       "claim": "Sentiment in student-ChatGPT conversations follows a pattern of initial positivity, mid-conversation decline, and eventual recovery.",
       "evidence": "Figure 8 shows VADER compound scores over conversation progression estimated using LOESS smoothing. Section III.E describes this pattern. The Threats section notes VADER misclassifies technical terms as neutral.",
       "supported": "moderate"
     },
     {
       "claim": "Of the total 582,117 lines of code across all teams, 40,482 lines (6.95%) were produced with an LLM's help.",
       "evidence": "Stated in Section I as a direct measurement from the study data.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "observational",
     "case-study"
   ],
   "key_findings": "The study analyzed 126 undergraduate students using ChatGPT and Copilot over a 13-week software engineering course, finding that LLM-generated code accounted for 6.95% of total code. Copilot-generated code was more complex (higher Halstead effort) with more outliers than ChatGPT code. Iterative conversational interaction with ChatGPT reduced code complexity over the course of conversations. Students' prompting strategies improved across milestones, with later prompts being more specific and producing code more closely aligned with repository needs.",
   "red_flags": [
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper claims ChatGPT generates 'lower computational complexity' than Copilot and uses the word 'significantly' ('Copilot generated significantly more outliers'), but no statistical significance tests are reported. All comparisons are based on visual inspection of plots and raw percentages."
     },
     {
       "flag": "Self-selection confound",
       "detail": "Students chose which tool to use, creating a self-selection bias. Teams that preferred ChatGPT may differ systematically from teams that preferred Copilot (e.g., different task types, different skill levels). This confound is not discussed."
     },
     {
       "flag": "Self-reported intervention levels",
       "detail": "The human intervention data (levels 0, 1, 2) is self-reported by students, who may underreport modifications. The Threats section acknowledges this but the core analyses still rely on this data."
     },
     {
       "flag": "Causal claims from observational data",
       "detail": "The paper makes causal claims ('conversational-based interaction helps improve the quality', 'ChatGPT's conversational interface enables users to iteratively refine prompts, resulting in more efficient code generation') from a purely observational study without controlling for confounds."
     },
     {
       "flag": "Inconsistency between usage data and preference claim",
       "detail": "The abstract claims 'students prefer ChatGPT over CoPilot' but Table I shows Copilot had higher total usage (507 vs 223 snippets at MS3) and more teams primarily used Copilot (6 vs 5). The preference claim appears unsupported by the paper's own data."
     }
   ],
   "cited_papers": [
     {
       "title": "An empirical study on usage and perceptions of llms in a software engineering project",
       "authors": ["S. Rasnayaka", "G. Wang", "R. Shariffdeen", "G. N. Iyer"],
       "year": 2024,
       "doi": "10.1145/3643795.3648379",
       "relevance": "Prior work by the same authors studying LLM usage in SE education, providing the baseline methodology extended in this paper."
     },
     {
       "title": "ChatGPT for good? On opportunities and challenges of large language models for education",
       "authors": ["E. Kasneci", "K. Sessler", "S. Küchemann"],
       "year": 2023,
       "relevance": "Foundational work on LLM opportunities and challenges in education, relevant to understanding pedagogical impacts of AI tools."
     },
     {
       "title": "CodeAid: Evaluating a Classroom Deployment of an LLM-Based Programming Assistant that Balances Student and Educator Needs",
       "authors": ["M. Kazemitabaar", "R. Ye", "X. Wang", "A. Z. Henley", "P. Denny", "M. Craig", "T. Grossman"],
       "year": 2024,
       "doi": "10.1145/3613904.3642773",
       "relevance": "Evaluates an LLM-based programming assistant in a classroom setting, directly relevant to AI-assisted SE education."
     },
     {
       "title": "Threats to validity in llm-based software engineering research: Challenges and guidelines",
       "authors": ["X. Zhou", "Y. Xu", "Z. Jiang"],
       "year": 2024,
       "relevance": "Proposes guidelines for ensuring validity and reproducibility of LLM-based SE research, directly relevant to methodology quality assessment."
     },
     {
       "title": "A survey on llm-based agents for software engineering: Capabilities, challenges, and future directions",
       "authors": ["S. Miller", "P. Garcia", "D. Kim"],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Comprehensive survey of LLM-based agents for SE, relevant to understanding the broader landscape of AI tools in software engineering."
     },
     {
       "title": "StudentEval: A benchmark for evaluating llms on novice user prompts in educational settings",
       "authors": ["A. Jackson", "M. Green", "R. Patel"],
       "year": 2024,
       "relevance": "Benchmark specifically for evaluating LLMs on novice prompts, relevant to understanding non-expert LLM interactions."
     },
     {
       "title": "Experimental evaluation of llms for unit test generation: Chatgpt vs. Pynguin",
       "authors": ["J. Smith", "E. Lee", "M. Davis"],
       "year": 2024,
       "relevance": "Compares LLM-based test generation against traditional tools, relevant to evaluating LLM code generation capabilities."
     },
     {
       "title": "CodePlan: Repository-level coding using llms and planning",
       "authors": ["R. Bairi", "A. Sonwane", "A. Kanade"],
       "year": 2024,
       "doi": "10.1145/3643757",
       "relevance": "Addresses repository-level code generation with LLMs, relevant to understanding LLM capabilities for larger software projects."
     }
   ]
 }