ai-research-survey

scan.json (29369B)

---

 {
   "paper": {
     "title": "How Beginning Programmers and Code LLMs (Mis)read Each Other",
     "authors": [
       "Sydney Nguyen",
       "Hannah McLean Babe",
       "Yangtian Zi",
       "Arjun Guha",
       "Carolyn Jane Anderson",
       "Molly Q Feldman"
     ],
     "year": 2024,
     "venue": "CHI '24 (Conference on Human Factors in Computing Systems)",
     "arxiv_id": "2401.15232",
     "doi": "10.1145/3613904.3642706"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["observational", "qualitative"],
   "key_findings": "In a controlled study of 120 CS1-complete students across 3 institutions attempting 48 CS1-level problems, participants achieved only a 57% eventual success rate and 24% per-attempt success rate when prompting Codex. Students' most common mental model of Code LLMs was keyword-based lookup, which led to unproductive strategies. Prior programming experience (p=0.02) and non-first-generation status (p=0.04) correlated with higher pass@1 rates. Students did not observably improve at prompting during the 75-minute study, and their most common editing strategy — adding detail — was not always effective.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper releases experimental data at https://doi.org/10.17605/OSF.IO/V2C4T but does not mention releasing the source code for the Charlie web application or the analysis scripts."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Data is publicly available at https://doi.org/10.17605/OSF.IO/V2C4T, including codebooks, supplemental materials, and collected experimental data. Stated in §1 footnote and Appendix A.2."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, library versions, etc.) is provided. The paper mentions using R's lme4 package and Python but does not give version details or reproducible environment specs."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the study methodology is described in detail (§3-4), there are no step-by-step reproduction instructions (e.g., scripts to replicate analyses, README with commands). The OSF repository contains data but the paper does not reference reproduction scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 1, 4) report point estimates of pass@1, success rate, and eventual success rate without confidence intervals or error bars. The mixed-effects model (Table 12) reports standard errors for coefficients, but the primary outcome measures lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Extensive use of statistical tests: Welch t-tests for demographic comparisons (Table 11), binomial mixed-effects model for category effects (Table 12), Kendall's tau and Pearson's r for correlations (§6.2, Tables 7-8). Significance level α=0.05 stated in §5.1."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are provided with context: pass@1 0.17 vs 0.24 for no-experience vs experienced students (§6.3), Kendall's tau values reported for NASA-TLX correlations (e.g., τ=-0.4 for self-rated success, §6.2), and percentage differences for success rates across categories (Table 4)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper argues its 120-participant sample is larger than prior work and enables statistical analyses (§3.1), but no formal power analysis is reported. The sample size appears driven by practical considerations (40 per institution) rather than statistical requirements."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "SDs are reported for some measures (time: SD=10.6, words added: SD=11.34), but the primary outcome measures (pass@1, success rates) in Tables 1 and 4 are reported as means without standard deviations or any spread measure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple prior studies: Prather et al. (19 students, Minesweeper), Kazemitabaar et al. (33 students, CodingSteps), Denny et al. (54 students, Promptly). Also includes a pilot with 19 more experienced students (68.8% eventual success rate) as an internal comparison (§4.4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparison papers are all from 2023, which are contemporary to the study period (data collected March-July 2023). The model used (code-davinci-002) is compared to contemporary alternatives like gpt-3.5-turbo and CodeLlama on HumanEval (§4.2)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational user study, not a system with components to ablate. All participants use the same interface and model."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: success rate, eventual success rate, pass@1 (§5.2), NASA-TLX workload ratings (Table 2), perception scales (Tables 7-8), plus qualitative analysis of interviews and strategies."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Students evaluated generated code via forced-choice questions ('Did Charlie generate correct code?', 'Would you have written this code yourself?' — §4.1). Additionally, two researchers performed thematic coding of interview responses and open-ended survey answers (§5.1, Appendix A.2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Problems have both provided test cases (shown to students as feedback) and expert test suites validated with code coverage and mutation testing (§3.2). Success is calculated 'using only the provided test cases' to align feedback with evaluation, but expert tests provide additional validation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides per-category breakdowns of pass@1 and eventual success rate across all 8 problem categories (Sorting, Dictionaries, Nested, Math, Loops, Lists, Conditionals, Strings). Institution-level breakdowns are also provided (Table 1, Figure 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis: §7.3 covers syntax errors (196 cases), model producing different programs from same prompt (107 cases), model producing same program despite prompt edits (104 cases). §7.1-7.2 analyze student-reported difficulties. Appendix B.2 provides detailed case studies of the hardest problems (laugh, total_bill) with all 20 student descriptions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper is primarily about negative results: students struggle with prompting (24% success rate), don't improve over time (§8.3), develop incorrect mental models (§8.1), and the most common editing strategy (adding detail) is not reliably effective. Code LLMs are found not to be a 'panacea for non-expert programming' (§9.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that beginners 'struggle with writing and editing prompts, even for problems at their skill level and when correctness is automatically determined.' This is supported by 57% eventual success rate and 24% per-attempt success rate (§6.1), and qualitative evidence throughout §7-8."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is carefully worded with correlational language: 'correlated with' (§6.3), 'may also be linked to' (§9.2). Stronger claims like 'Code LLMs remain inaccessible to non-experts' (§11) are supported by the controlled experimental evidence. The study design adequately supports its descriptive and correlational claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "§5.3 explicitly states: 'This research studies students at three selective higher education institutions in the United States... our findings may not generalize to other settings (e.g., community colleges, K-12 education) or cultural contexts.' The specific model and its limitations are also discussed (§9.5)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§10 (Threats to Validity) discusses specific alternatives: ChatGPT awareness during data collection, non-homogeneous programming backgrounds, potential bias from anthropomorphic system design, novelty bias, and self-selection bias. §5.3 (Positionality) discusses researcher incentives."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes what it measures (success rate, eventual success rate, pass@1) from broader claims. §5.2 explains why pass@1 is preferable to raw success rate for measuring prompt quality (accounts for LLM stochasticity). Claims are framed at the level of the measurements rather than extrapolated to broader constructs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The specific Codex model is named: 'code-davinci-002' (§4.2). StarCoder is named for pass@1 resampling (§5.2). The paper also contextualizes these models with HumanEval scores (46% and comparable to gpt-3.5-turbo at 48%)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows the exact prompt format sent to Codex (function signature + student description as docstring). Table 13 provides all 20 initial student descriptions for one problem. All prompts are available in the released dataset at OSF."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We generated output using best practices for hyperparameter and sampler settings [13]' referencing Chen et al. (2021), but does not state the actual temperature, top-p, or sampling parameters used. A 256 token limit is mentioned (§7.3.1) but other settings are not."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a simple prompt-to-completion pipeline: student description → formatted as docstring → Codex API call → code returned (Figure 4)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Problem adaptation from CS1 materials is described (Appendix A.1), problem validation procedure documented (A.1.3), test case validation with mutation testing described (A.1.4), qualitative coding methodology detailed (A.2), and the pass@1 computation procedure explained (§5.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 'Threats to Validity' is a dedicated section spanning approximately one full page, discussing multiple specific threats to the study's validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§10 discusses specific threats: ChatGPT release between pilot and main study affecting student awareness, heterogeneous programming backgrounds despite eligibility criteria, power dynamics between students and professors, anthropomorphic system design potentially biasing positive perceptions, and model deprecation risk."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§5.3 explicitly states findings may not generalize to community colleges, K-12, or other cultural contexts. §9.5 discusses model-specific limitations. §9.6 notes the study captures a specific temporal moment before widespread LLM familiarity. The paper distinguishes its simplified task (automated testing) from the full real-world prompting task."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Raw data is publicly available at https://doi.org/10.17605/OSF.IO/V2C4T, referenced in §1 footnote and Appendix A.2. Supplemental materials including codebooks are also provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed description of the Charlie web application (§4.1, Figure 3-4), Zoom-recorded sessions with audio/video (§4.4), post-task survey and semi-structured interviews with specific questions described (§4.4), and data collection timeline (March-July 2023)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "§4.3: 'We recruited 40 participants from each institution (n=120). Eligible participants were at least 18 years old, had taken CS1 at their institution between Fall 2021 and Spring 2023, and had not completed any subsequent CS courses.' Recruitment via interest form distributed by other faculty/staff, scheduling by a researcher at another institution."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: recruitment → consent/assent → 3 tutorial problems → 8 main problems (4 untimed, 4 timed) → post-survey part 1 → semi-structured interview → post-survey part 2 → debrief (Figure 2). Qualitative coding pipeline detailed in Appendix A.2 with multiple rounds of coding towards consensus."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section: 'This work is partially supported by the National Science Foundation (SES-2326173, SES-2326174, and SES-2326175).' Computing resources from Northeastern Research Computing and New England Research Cloud also acknowledged."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Wellesley College, Oberlin College, Northeastern University, and Roblox (for Arjun Guha). §5.3 (Positionality) notes 'Some authors also contribute to the development and evaluation of open-source Code LLMs.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSF, which has no financial stake in whether Code LLMs are found effective or not. The study does not evaluate any product made by the funders or authors' employers."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No formal competing interests statement is included. §5.3 notes some authors contribute to open-source Code LLMs and the research team has 'complex incentives,' but there is no explicit declaration of patents, equity, or other financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a human interaction study examining how students prompt Code LLMs, not a benchmark evaluation of model capability. The model (Codex) is a tool in the study, not the subject of evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation study. The research questions are about human prompting behavior, not model performance on benchmarks. However, the paper does validate that problems cannot be solved from function names alone (§3.3, A.1.3)."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation study. The study examines human-AI interaction, not model capability on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, or similar). The OSF link (https://doi.org/10.17605/OSF.IO/V2C4T) is for data sharing, not pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "§4.4 states: 'The pilot and main study received IRB approval.' Informed consent procedures are also described."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Extensive demographics: gender (Table 9), race (Table 10), first-generation status, high school type, household language, international status (Table 1), programming experience, math courses, and major (§6.3). Open-ended demographic questions following best practices (§4.4, reference to Spiel et al.)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "§4.3: 'Eligible participants were at least 18 years old, had taken CS1 at their institution between Fall 2021 and Spring 2023, and had not completed any subsequent CS courses.' Recruitment continued until sample size of 120 reached."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "§4.4: 'Participants were randomly assigned experimental lists, balanced by difficulty, using a modified Latin Square design. Four authors independently assessed the difficulty of writing prompts for each problem; we averaged these scores and developed six roughly equal lists.' Problem assignment ensures each problem gets 20 students."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a single-condition within-subjects design — all participants use the same system (Charlie/Codex). There are no between-subjects treatment conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports n=120 throughout, implying no participant dropout. Problem-level attrition is documented: Figure 5d shows attempts ending in success vs give-up, 340 problems where students gave up are reported (§7.3.3). Five missing interview responses out of 960 possible are noted (§4.4)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a human subjects user study, not a system paper proposing a deployable method. Cost of the Codex API calls and StarCoder computation is not the focus."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a human subjects study. The computational cost of running the study (API calls, StarCoder resampling) is not the focus; the study evaluates human interaction, not computational efficiency."
       }
     }
   },
   "claims": [
     {
       "claim": "Beginning programmers who have completed CS1 struggle to prompt Code LLMs effectively, achieving only 57% eventual success rate and 24% per-attempt success rate on CS1-level problems.",
       "evidence": "§6.1, Figure 5: 120 students across 3 institutions solved 4.7/8 problems on average, with mean eventual success rate of 57% and mean success rate of 24%. Mean pass@1 of 0.22.",
       "supported": "strong"
     },
     {
       "claim": "Prior programming experience outside CS1 is positively correlated with prompting success (pass@1 of 0.24 vs 0.17).",
       "evidence": "§6.3, Table 11: Welch t-test p=0.02 for students with additional coding experience vs CS1-only students.",
       "supported": "strong"
     },
     {
       "claim": "First-generation college students struggle more with Code LLM prompting than non-first-generation students.",
       "evidence": "§6.3, Table 11: pass@1 of 0.17 vs 0.23 with p=0.04 (Welch t-test). N=23 first-generation vs 96 non-first-generation.",
       "supported": "moderate"
     },
     {
       "claim": "Students do not observably improve at prompting Code LLMs during the 75-minute study.",
       "evidence": "§8.3: Comparison of success rates for students attempting each problem first vs last (5 students each) shows no significant difference.",
       "supported": "moderate"
     },
     {
       "claim": "Students' most common mental model of Code LLMs is keyword-based lookup, which is incorrect and may inhibit effective strategy development.",
       "evidence": "§8.1, Table 5: 46 of ~96 responding students described a keyword-based model. Students with correct ChatGPT/LLM mental models had slightly higher success rates (0.27 vs 0.22, p=0.03).",
       "supported": "moderate"
     },
     {
       "claim": "The most common prompting strategy is adding detail, with students adding an average of 9.44 words between first and last prompts.",
       "evidence": "§8.2.1, Table 6, Figure 7: 48 students mentioned adding detail. Quantitative analysis of 282 successful multi-attempt prompts confirms mean addition of 9.44 words (SD=11.34).",
       "supported": "strong"
     },
     {
       "claim": "Model stochasticity significantly impacts student experience: same prompt produces different code (107 cases, 4.2%), and different prompts produce same code (104 cases, 11% of edits).",
       "evidence": "§7.3.2-7.3.3: 107 cases of identical prompt resubmission with different outputs; 104 submissions where edited prompts produced identical code, occurring in 36/48 problems and affecting 72/120 students.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Multiple comparisons without correction",
       "detail": "Table 11 reports 6 independent t-tests on demographic subgroups without Bonferroni or other multiple comparison corrections. The first-generation finding (p=0.04) would not survive correction for 6 comparisons (adjusted threshold ~0.008). This weakens the equity claim."
     },
     {
       "flag": "Selective institution sample",
       "detail": "All three institutions are selective US colleges/universities (an R1 university, a liberal arts college, a women's college). The authors acknowledge this in §5.3 but the findings may not generalize to the broader population of beginning programmers (community colleges, bootcamps, non-US contexts)."
     },
     {
       "flag": "No formal power analysis",
       "detail": "The 120-participant sample size appears driven by practical constraints (40 per institution) rather than statistical power requirements. Some subgroup analyses (e.g., 23 first-generation students, 6 Black participants) may be underpowered."
     }
   ],
   "cited_papers": [
     {
       "title": "Grounded Copilot: How Programmers Interact with Code-Generating Models",
       "authors": ["Shraddha Barke", "Michael B. James", "Nadia Polikarpova"],
       "year": 2023,
       "doi": "10.1145/3586030",
       "relevance": "Studies how experienced programmers interact with Copilot, identifying modes of interaction — key comparison study for expert vs novice LLM use."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "The Codex paper that introduced HumanEval and pass@1 metrics used in this study; foundational for Code LLM evaluation methodology."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "RCT studying Copilot's impact on developer productivity; relevant to the productivity paradox in AI-assisted programming."
     },
     {
       "title": "Expectation vs. Experience: Evaluating the Usability of Code Generation Tools Powered by Large Language Models",
       "authors": ["Priyan Vaithilingam", "Tianyi Zhang", "Elena L. Glassman"],
       "year": 2022,
       "doi": "10.1145/3491101.3519665",
       "relevance": "Earliest academic usability study of Copilot; found users enjoyed it but it didn't help them code faster or more correctly."
     },
     {
       "title": "\"It's Weird That it Knows What I Want\": Usability and Interactions with Copilot for Novice Programmers",
       "authors": ["James Prather", "Brent N. Reeves", "Paul Denny"],
       "year": 2023,
       "doi": "10.1145/3617367",
       "relevance": "Studies 19 CS1 students using Copilot for a final project; found students struggled and often mistakenly accepted incorrect code."
     },
     {
       "title": "Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming",
       "authors": ["Majeed Kazemitabaar", "Justin Chow", "Carl Ka To Ma"],
       "year": 2023,
       "doi": "10.1145/3544548.3580919",
       "relevance": "Studies 33 K-12 students with Codex access vs 36 without; found Code LLMs may benefit learning but students heavily reused expert descriptions."
     },
     {
       "title": "Promptly: Using Prompt Problems to Teach Learners How to Effectively Utilize AI Code Generators",
       "authors": ["Paul Denny", "Juho Leinonen", "James Prather"],
       "year": 2023,
       "arxiv_id": "2307.16364",
       "relevance": "Studies 54 students writing prompts for 3 CS1 problems; found similar challenges to this paper at smaller scale."
     },
     {
       "title": "StudentEval: A Benchmark of Student-Written Prompts for Large Language Models of Code",
       "authors": ["Hannah McLean Babe", "Sydney Nguyen", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2306.04556",
       "relevance": "Companion benchmark paper using student-written prompts from this study to evaluate Code LLMs."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open Code LLM used in this study for pass@1 resampling; relevant to open-source model alternatives for reproducibility."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "GPT-4's HumanEval performance (67%) provides context for the capability level of models used in this study."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Open Code LLM achieving comparable performance to Codex on HumanEval; supports the study's argument about model-agnostic findings."
     },
     {
       "title": "Why Johnny Can't Prompt: How Non-AI Experts Try (and Fail) to Design LLM Prompts",
       "authors": ["J.D. Zamfirescu-Pereira", "Richmond Y. Wong", "Bjoern Hartmann", "Qian Yang"],
       "year": 2023,
       "doi": "10.1145/3544548.3581388",
       "relevance": "Studies non-expert LLM prompting for non-code tasks (recipes); found similar issues with incorrect mental models and ineffective strategies."
     },
     {
       "title": "GitHub Copilot AI pair programmer: Asset or Liability?",
       "authors": ["Arghavan Moradi Dakhel", "Vahid Majdinasab"],
       "year": 2022,
       "doi": "10.48550/ARXIV.2206.15331",
       "relevance": "Compares quality of Codex-generated code to student-written code; relevant to understanding Code LLM quality for educational contexts."
     },
     {
       "title": "CodeCompose: A Large-Scale Industrial Deployment of AI-assisted Code Authoring",
       "authors": ["Vijayaraghavan Murali", "Chandra Maddila"],
       "year": 2023,
       "arxiv_id": "2305.12050",
       "relevance": "Large-scale industrial study of AI-assisted coding finding that it enhances expert programmer productivity; contrasts with beginner struggles."
     }
   ]
 }