ai-research-survey

scan.json (30946B)

---

 {
   "paper": {
     "title": "No Need to Lift a Finger Anymore? Assessing the Quality of Code Generation by ChatGPT",
     "authors": [
       "Zhijie Liu",
       "Yutian Tang",
       "Xiapu Luo",
       "Yuming Zhou",
       "Liang Feng Zhang"
     ],
     "year": 2023,
     "venue": "IEEE Transactions on Software Engineering",
     "arxiv_id": "2308.04838",
     "doi": "10.1109/TSE.2024.3392499"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ChatGPT (GPT-3.5-turbo-0301) generates functionally correct code at 68.41% acceptance rate for pre-2021 LeetCode problems but only 20.27% for post-2021 problems, a 48.14% gap attributed to training data exposure. Multi-round fixing is largely ineffective for functional correctness (<32% fix rate) but highly effective for security vulnerabilities (89-100% fix rate). Code complexity tends to increase through multi-round fixing. The paper identifies specific defect categories (WD, MCC, MP) and shows ChatGPT's non-determinism significantly affects all quality dimensions at default temperature.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The experimental scripts, results, and raw data are available at: [28]' pointing to a Zenodo archive (https://zenodo.org/records/10556350)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Raw data is released via the same Zenodo archive [28]. Additionally, the datasets used (LeetCode problems and CWE scenarios from [23]) are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 specifies: Intel i9-10900X CPU @ 3.70GHz, 128GB RAM, Ubuntu 20.04, Python 3.10.9, CodeQL 2.12.2, and the specific ChatGPT model version gpt-3.5-turbo-0301 with temperature 0.7."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Experimental scripts are provided at the Zenodo artifact [28]. The methodology sections (Sec 3-4) describe the workflow, prompt design, and evaluation process in sufficient detail for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 1-2, 5, 8, 10, 11, 18-21) report only point estimates (percentages) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Wilcoxon rank-sum test is used throughout with p-values reported (e.g., p=0.008 for before/after 2021 comparison). Holm-Bonferroni correction is applied for multiple comparisons (Sec 4.1)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cliff's Delta effect size measure is used alongside Wilcoxon tests. Effect size values are reported (e.g., effect size value of 1 for period comparison, 0.3125 for C.E. rate comparison between periods). Percentage differences with context are also provided (e.g., '48.14% advantage')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "728 algorithm problems are used (354 after 2021, 374 sampled before 2021) but no power analysis or explicit justification for why these specific numbers were chosen. The sampling follows the difficulty distribution ratio (1:2:1) but no statistical justification for total sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main experiments query each prompt once. The paper acknowledges ChatGPT's non-determinism (RQ5) and runs 10 trials for a subset of 18 problems, but the main results in Tables 1-2 are single-run with no variance measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human-written code from LeetCode is used as a baseline for complexity comparison (Tables 14-16). Human acceptance rates are compared with ChatGPT acceptance rates. Before/after 2021 comparison serves as a contamination baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No contemporary AI code generation models (Codex, CodeGen, Copilot, etc.) are compared. The only baselines are human-written solutions and human acceptance rates. For a 2023 paper evaluating code generation, omitting comparison with other LLM-based tools is a gap."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a single pre-existing system (ChatGPT) with no components to ablate. It is not proposing a new multi-component method."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Accepted rate, Wrong Answer rate, Compile Error rate, Runtime Error rate, Time Limit Exceeded rate, test case pass rate, cyclomatic complexity, cognitive complexity, and vulnerability counts across multiple CWE categories."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two graduate students independently classify W.A. code defects (157 pairs, 0.7325 consistency ratio, with senior analyst for disagreements). Manual analysis is also performed for T.L.E. causes, C.E. classification, vulnerability verification, and Type-2 clone detection."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Post-2021 problems (354) serve as a held-out test set that could not have appeared in ChatGPT's training data (trained on data before 2021). Pre-2021 problems are explicitly acknowledged as potentially in the training set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns by language (C, C++, Java, Python3, JavaScript), difficulty level (easy, medium, hard), time period (before/after 2021), error type (W.A., C.E., R.E., T.L.E.), defect class (Table 4), compile error class (Table 7), runtime error class (Table 9), and CWE vulnerability groups (Table 20)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis: three error categories (WD, MCC, MP) with code examples (Figs. 4, 6, 8-19), analysis of unfixed multi-round cases (Table 6), EIL and EBL error categories for compile errors, and specific vulnerability examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: ChatGPT's weak multi-round fixing ability (<32% fix rate for W.A.), poor performance on hard problems (0.66% A. rate for hard Aft. problems), persistent vulnerability patterns, complexity increase through fixing, and unfixable CWE-20 vulnerabilities."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: 48.14% advantage (Tables 1-2), weak multi-round fixing (Tables 5, 8, 10, 11), complexity variation (Tables 12-13, Fig. 25), >89% vulnerabilities fixed (Tables 18, 21), and non-determinism effects (Tables 22-32)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is primarily descriptive/evaluative. Where comparative claims are made (before vs after 2021), Wilcoxon rank-sum tests with effect sizes are used. Claims about multi-round fixing effects are based on before/after measurement within the same conversation. No strong unsupported causal claims are made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its scope: specifies the model (gpt-3.5-turbo-0301), the benchmark (LeetCode algorithm problems), languages tested (5 specified), and CWE scenarios. Threats to validity (Sec 5.3) explicitly acknowledges 'LeetCode problems... may not fully represent the complexity and diversity of real-world coding tasks.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Several alternatives are addressed: (1) post-2021 problems might be reformulations of pre-2021 ones (checked via 'similar questions' feature and manual analysis of 142 pairs, ruled out), (2) non-determinism as confound (RQ5, temperature 0 analysis), (3) token limitation impact (Sec 5.1, simulated incomplete generation)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures functional correctness (LeetCode acceptance), complexity (cyclomatic/cognitive), and security (CodeQL detection), and frames results at the same granularity — 'code generation quality' across these three specific dimensions. No broader proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Sec 3.2 states: 'we utilize the model version gpt-3.5-turbo-0301 of ChatGPT' — exact API version specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt examples are provided: Fig. 3 shows a complete prompt with all four components (<Content>, <Examples>, <Template>, <Command>) using a real problem. Multi-round fixing prompt examples are shown throughout Sec 4.2 with actual error messages."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 0.7 (default, Sec 3.2), token limitation 4,096 (Sec 3.2), and the token-limitation strategy for multi-round conversations are reported. Temperature 0 is also tested in RQ5."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-round workflow is described in detail with a diagram (Fig. 2): prompt construction → ChatGPT response → testing (LeetCode/CodeQL) → error feedback → re-prompting, including the token-limitation strategy for managing conversation context."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data collection and preprocessing are well-documented: problem sampling strategy (1:2:1 difficulty ratio), before/after 2021 split criteria (2022-01-01 divider), CWE scenario sources, prompt construction process, code extraction from responses (between triple backticks), and filtering of constant function outputs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated 'Limitations' section (Sec 5.2) discussing closed-source model constraints and evolving model concerns, plus a 'Threats to Validity' section (Sec 5.3) with five specific threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: LeetCode problems may not represent real-world coding, LeetCode terminates testing at first failure (affecting pass rates), CodeQL may report false positives (mitigated by manual inspection), limited language coverage for vulnerability detection, and ChatGPT's randomness (Sec 5.3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'LeetCode problems are designed specifically for coding practice and interview preparation... they may not fully represent the complexity and diversity of real-world coding tasks' (Sec 5.3). The model is bounded to GPT-3.5-turbo-0301 and its limitations are noted (Sec 5.2)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The experimental scripts, results, and raw data are available at: [28]' (Zenodo archive https://zenodo.org/records/10556350)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed collection description: 728 algorithm problems from LeetCode (354 after 2021, 374 sampled before 2021 following 1:2:1 difficulty ratio), 18 CWEs with 54 scenarios from MITRE Top 25 CWEs and [23]. Sampling criteria, time divider, and problem selection process documented in Sec 3.1 and 4.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Data sources are standard benchmarks (LeetCode platform, CWE scenarios from MITRE and prior work). Graduate student annotators are internal researchers, not recruited participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: problem collection → prompt construction → ChatGPT querying → code extraction → LeetCode judgment/CodeQL analysis → error classification → multi-round fixing. Each step is described with clear criteria and counts (e.g., 1,870 + 1,770 prompts generated)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Funding sources are not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: ShanghaiTech University, University of Glasgow, Hong Kong Polytechnic University, and Nanjing University. Authors are academic researchers evaluating a third-party product (ChatGPT/OpenAI)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors are from academic institutions with no apparent affiliation with OpenAI, but without explicit disclosure, this criterion is not met."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Sec 4.1 states 'ChatGPT is trained on text data before 2021' and uses this as the basis for the before/after 2021 experimental split."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensively discussed: the paper acknowledges 'Bef. problems and corresponding solutions may have a high probability to appear in its training set' (Sec 4.1). They manually check 142 similar question pairs between periods. They find 6/250 solutions are Type-2 clones to ground truth, indicating ~5% training data memorization."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire experimental design splits before/after 2021 to address contamination. The 48.14% performance gap between periods is attributed partly to training data exposure. Type-2 clone analysis provides direct evidence of memorization."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Graduate students serve as code analysts/annotators but are not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates an AI system on public benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Graduate student annotators are described only as 'two graduate students with experience in algorithm analysis.'"
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token consumption, or per-query latency are reported despite making thousands of API calls (3,640+ prompts in the main experiment plus multi-round fixing and non-determinism studies)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (Sec 3.3) but total computational budget (wall-clock time, total API spend, number of tokens consumed) is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main experiments (RQ1-RQ4) query each prompt once. RQ5 explores non-determinism with 10 trials on 18 problems, showing significant variation at temperature 0.7, but this is a separate analysis on a small subset rather than seed sensitivity for the main results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Sec 4.1: 'we input every prompt once into it to ask for generating code.' RQ5 explicitly states 10 trials. The number of runs is clear throughout."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Default temperature 0.7 is used with no hyperparameter search. Temperature 0 is tested only in RQ5 for non-determinism analysis. No systematic search over prompt designs or model parameters is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Sec 3.2 explicitly states: 'The goal of our prompt design is not to find the optimal prompt that maximizes ChatGPT's performance. Instead, our goal is to provide a reasonable prompt that simulates real-world usage scenarios.' Default temperature 0.7 is used to 'simulate real-world usage scenarios.'"
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Holm-Bonferroni correction is applied for multiple comparisons: 'In cases of multiple comparisons, we apply Holm-Bonferroni correction, a commonly used technique, to adjust p-values' (Sec 4.1)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not discuss potential biases in their evaluation design (e.g., whether their prompt template advantages or disadvantages ChatGPT, or whether their defect classification might be biased)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Multi-round fixing uses progressively more API calls (up to 5 or 10 rounds) but performance is not reported as a function of compute cost. No cost comparison between one-round and multi-round approaches."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Sec 5.3 discusses construct validity: 'LeetCode problems are designed specifically for coding practice and interview preparation... they may not fully represent the complexity and diversity of real-world coding tasks. Real-world coding scenarios often involve various external factors, domain-specific requirements.' CWE scenarios are also acknowledged as potentially incomplete."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a single model (ChatGPT) with a consistent prompting approach. No multi-model comparison across different scaffolds is performed, so the scaffold confound does not apply."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The before/after 2021 split is explicitly designed to address temporal leakage. The paper demonstrates a 48.14% performance gap consistent with training data exposure, and manually checks for Type-2 clones to detect memorization."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The prompts include method signatures and examples from LeetCode which could provide hints. Error messages in multi-round fixing provide specific feedback, but this is the intended design rather than accidental leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper checks for similarity between before/after 2021 problems using LeetCode's 'similar questions' feature, finding 142 pairs. Manual analysis by two graduate students confirms these are 'either having similar scenarios but completely different solution goals, or different scenarios and conditions but can be solved using similar algorithms' (Sec 4.1)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Type-2 clone detection is applied: 50 sampled problems (250 solutions) are compared against 5 ground truth solutions each. 6 solutions in easy/medium difficulties were found to be Type-2 clones, providing direct evidence of training data memorization (Sec 4.1)."
       }
     }
   },
   "claims": [
     {
       "claim": "ChatGPT generates functionally correct code for pre-2021 problems at 48.14% higher Accepted rate than post-2021 problems (68.41% vs 20.27%)",
       "evidence": "Tables 1-2 show per-language, per-difficulty status rates across 1,870 and 1,770 prompts for before and after 2021 problems. Wilcoxon rank-sum test p=0.008, Cliff's Delta effect size=1 (Sec 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Multi-round fixing can only fix a small fraction (<32%) of erroneous code to Accepted status",
       "evidence": "Table 5: 25/157 (15.9%) W.A. pairs fixed. Table 8: 40/155 (26%) C.E. pairs fixed to A. Table 10: 52/194 (27%) R.E. pairs fixed to A. Table 11: 44/140 (31.4%) T.L.E. pairs fixed (Sec 4.2).",
       "supported": "strong"
     },
     {
       "claim": "The multi-round fixing process generally preserves or increases code complexity levels",
       "evidence": "Figure 25 shows heatmaps where diagonal cells (same complexity) exceed 50% in all language-complexity combinations, and cells above the diagonal (increased complexity) consistently have higher percentages than below-diagonal cells (Sec 4.3).",
       "supported": "moderate"
     },
     {
       "claim": "ChatGPT-generated code has security vulnerabilities (33.32% of valid CWE scenario code) but multi-round fixing addresses 89.4% of them",
       "evidence": "Table 19: 994/2,983 valid CWE code snippets are vulnerable. Table 21: 143/160 sampled vulnerable snippets fixed. Table 18: all sampled algorithm problem vulnerabilities fixed (Sec 4.4).",
       "supported": "strong"
     },
     {
       "claim": "ChatGPT's non-determinism affects code generation quality, but temperature 0 mitigates this in one-round process",
       "evidence": "Tables 22-25: at temperature 0.7, same problem-language pairs produce different statuses, complexity levels, and vulnerability rates across 10 trials. Tables 24-26: at temperature 0, one-round outputs are identical across all trials except one case (Sec 4.5).",
       "supported": "strong"
     },
     {
       "claim": "Code generated in C has higher complexity and lower acceptance rates than other languages (C++, Java, Python3, JavaScript)",
       "evidence": "Table 12: C has 54.9% low+moderate cyclomatic complexity vs 66%+ for other languages. Tables 1-2: C acceptance rate of 31.28% overall vs 44-50% for other languages (Sec 4.1, 4.3).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Single model evaluation without competing models",
       "detail": "The paper evaluates only ChatGPT (GPT-3.5-turbo-0301) without comparing against other code generation models (Codex, CodeGen, Copilot, GPT-4). This limits the ability to contextualize ChatGPT's performance relative to the state of the art."
     },
     {
       "flag": "Single-run main experiment despite documented non-determinism",
       "detail": "RQ5 demonstrates significant non-determinism at temperature 0.7 (the default used), yet the main experiments (RQ1-RQ4) query each prompt only once. Tables 22-23 show acceptance rates varying from 0% to 100% across trials for the same problem, suggesting main results could be unreliable."
     },
     {
       "flag": "No cost reporting for extensive API usage",
       "detail": "The study makes thousands of API calls (3,640+ prompts in the main experiment, plus multi-round fixing and non-determinism studies) without reporting any cost, token consumption, or time metrics."
     },
     {
       "flag": "Outdated model version",
       "detail": "Uses gpt-3.5-turbo-0301 (March 2023 snapshot). GPT-3.5 and GPT-4 have been substantially updated since. Results may not generalize to current model versions, as acknowledged in Sec 5.2."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark for evaluating LLM code generation — foundational benchmark-eval paper."
     },
     {
       "title": "Asleep at the keyboard? assessing the security of github copilot's code contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan", "B. Dolan-Gavitt", "R. Karri"],
       "year": 2022,
       "relevance": "Evaluates Copilot's security code generation using CWE scenarios — directly comparable methodology for assessing LLM code security."
     },
     {
       "title": "An empirical evaluation of github copilot's code suggestions",
       "authors": ["N. Nguyen", "S. Nadi"],
       "year": 2022,
       "relevance": "Evaluates Copilot's code suggestion quality across 33 LeetCode problems — earlier benchmark-eval study of LLM code generation."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Proposes EvalPlus framework for more rigorous evaluation of LLM code generation, addressing benchmark adequacy."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Z. Fan", "X. Gao", "A. Roychoudhury", "S. H. Tan"],
       "year": 2022,
       "arxiv_id": "2205.10583",
       "relevance": "Studies whether APR techniques including Codex can fix incorrect LLM-generated code — related to multi-round fixing evaluation."
     },
     {
       "title": "Self-collaboration code generation via chatgpt",
       "authors": ["Y. Dong", "X. Jiang", "Z. Jin", "G. Li"],
       "year": 2023,
       "arxiv_id": "2304.07590",
       "relevance": "Proposes multi-role ChatGPT framework for code generation, relevant to agentic LLM workflows."
     },
     {
       "title": "Examining zero-shot vulnerability repair with large language models",
       "authors": ["H. Pearce", "B. Tan", "B. Ahmad", "R. Karri", "B. Dolan-Gavitt"],
       "year": 2023,
       "relevance": "Evaluates LLMs for zero-shot vulnerability repair — directly related to security code fixing evaluation."
     },
     {
       "title": "Codegen: An open large language model for code with multi-turn program synthesis",
       "authors": ["E. Nijkamp", "B. Pang", "H. Hayashi"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "Open LLM for multi-turn code generation — relevant baseline for code generation capability evaluation."
     },
     {
       "title": "Measuring coding challenge competence with apps",
       "authors": ["D. Hendrycks", "S. Basart", "S. Kadavath"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "APPS benchmark for evaluating code generation using programming challenges — related benchmark approach."
     },
     {
       "title": "Refining chatgpt-generated code: Characterizing and mitigating code quality issues",
       "authors": ["Y. Liu", "T. Le-Cong", "R. Widyasari"],
       "year": 2023,
       "arxiv_id": "2307.12596",
       "relevance": "Characterizes code quality issues in ChatGPT-generated code including correctness and maintainability — closely related study."
     },
     {
       "title": "Choose your programming copilot: A comparison of the program synthesis performance of github copilot and genetic programming",
       "authors": ["D. Sobania", "M. Briesch", "F. Rothlauf"],
       "year": 2022,
       "relevance": "Compares Copilot against genetic programming for code synthesis — benchmark-eval of AI code generation."
     },
     {
       "title": "Security weaknesses of copilot generated code in github",
       "authors": ["Y. Fu", "P. Liang", "A. Tahir"],
       "year": 2023,
       "arxiv_id": "2310.02059",
       "relevance": "Studies security weaknesses in Copilot-generated code found in GitHub repositories — real-world security evaluation."
     }
   ]
 }