ai-research-survey

scan.json (30889B)

---

 {
   "paper": {
     "title": "The Right Prompts for the Job: Repair Code-Review Defects with Large Language Model",
     "authors": [
       "Zelin Zhao",
       "Zhaogui Xu",
       "Jialong Zhu",
       "Peng Di",
       "Yuan Yao",
       "Xiaoxing Ma"
     ],
     "year": 2023,
     "venue": "ICSE 2024",
     "arxiv_id": "2312.17485",
     "doi": "10.48550/arXiv.2312.17485"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeLLaMA finetuned with the most informative prompt (combining review comments and fix range) achieves 72.97% exact-code-match repair rate on code review defects, a 25pp improvement over the baseline prompt with no hints. Fix range information provides the largest single improvement (avg +7.69pp), while cross-dataset transferability is poor—models finetuned on reviewer comments lose ~50% accuracy when tested on automated checker defects and vice versa. The 6-7B parameter range offers the best efficiency-effectiveness tradeoff across the CodeT5+ model family.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper. No mention of releasing the finetuning code or evaluation scripts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The reviewer dataset (RD) is sourced from Tufano et al. [32] which is public, but the PMD dataset (PD) of 14,935 entries was custom-collected by the authors and is not released. No download link or archive is provided for PD."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (2x NVIDIA A100-SXM4-80GB, AMD EPYC 7T83 CPU, 200GB memory) and HuggingFace transformers library is referenced, but no software versions, requirements.txt, Dockerfile, or detailed dependency list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. A reader would have to reverse-engineer the experimental setup from the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, and 5 are reported as point estimates (e.g., '72.97%') with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'CodeLLaMA outperformed the others', 'ChatGPT-4 surpasses ChatGPT-3.5') based solely on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as absolute ECM improvements with baseline context. Table 4 shows improvements between adjacent prompts (e.g., average +7.69pp from P4 to P5). The last row of Table 2 shows total improvement (#P7-#P3) for each model."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for the dataset sizes (~16K reviewer comments, ~15K PMD entries). The RD dataset size is inherited from Tufano et al., and the PD dataset was sized 'to maintain a similar magnitude' without further justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported. There is no mention of multiple training runs or seeds. All results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Nine models are compared (ChatGPT-3.5, ChatGPT-4, InCoder, CodeT5+, CodeFuse, LLaMA, CodeGen-2, CodeLLaMA, CodeReviewer). Zero-shot ChatGPT serves as a baseline, and the minimal prompt P3 serves as the baseline prompt configuration."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models were contemporary at the time of writing (2023): ChatGPT-4 (March 2023), CodeLLaMA (2023), LLaMA (2023), CodeGen-2 (2023). These represent the state of the art for code generation at that time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The prompt comparison (P3→P7) is a systematic ablation that incrementally adds information components (bug location, fix range, review comments). Table 4 quantifies the contribution of each added component. RQ4 ablates the dataset composition."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: exact-code-match (ECM) based on AST comparison, and Code BLEU. Both are reported in Tables 1 and 2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated patches is performed. Evaluation is entirely automated using ECM and Code BLEU. The authors acknowledge that ECM may miss semantically correct but syntactically different fixes, but do not address this with human review."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The datasets are split 9:1 into training and validation sets (28,063 training, 3,100 validation). The same hyperparameters are used across all models ('the same hyperparameters, including two training epochs, a learning rate of 2e-4, and a batch size of 16'), so the validation set is not used for tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-defect-type ECM breakdowns for all 30 PMD defect types, showing lowest and highest scores across all finetuning configurations. This reveals substantial variation (0% to 100%)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 shows an incorrect fix generated by CodeLLaMA. Figure 6 and Table 3 discuss the 'catch exception when parsing double' defect type with 0-6.7% fix rate. The paper explains why certain defect types are hard to fix."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: ChatGPT-4 performs worse than ChatGPT-3.5 on P1 (6.61% vs 14.10%), cross-dataset transferability is poor (~50% drop in Table 5), CodeLLaMA-13B shows a minor decrease vs 7B (-0.29%), and some defect types have 0% fix rate."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's main claim of '72.97% repair rate with the best prompt' is directly supported by Table 2 (CodeLLaMA with P7). The claim of 'substantial improvement in effectiveness' is supported by the P3→P7 comparison showing >20pp gains."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary causal claims concern prompt components causing performance changes. The experimental design—same models tested with systematically varying prompts (P3-P7)—is a controlled single-variable manipulation adequate for these causal claims. Table 4 quantifies each component's contribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests exclusively on Java code (PMD checker for Java, reviewer dataset primarily Java from GitHub/Gerrit), but the title and claims generalize to 'Code-Review Defects' without language qualification. The abstract makes no mention of the Java-only scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Threats section (Section 5) is brief and does not discuss specific alternative explanations for the results. For instance, it does not consider whether the results might be driven by data memorization, the simplicity of PMD-type fixes, or the quality of review comments rather than model capabilities."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "ECM measures syntactic equivalence (via AST comparison) but the paper frames this as 'repair rate' and 'fix rate.' They acknowledge that string comparison is unreliable and use AST-based comparison, but do not discuss that semantically correct fixes with different syntax are counted as failures, inflating the actual failure rate."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "ChatGPT-3.5 and ChatGPT-4 are referenced by marketing names without API version strings or snapshot dates. Open-source models specify parameter counts (e.g., 'CodeLLaMA 7B', 'CodeT5+ 6B') but not specific checkpoint versions or release dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided for all 7 prompts in Section 3.2. Figure 4 shows the prompt template, and each prompt variant (P1-P7) has its exact task instruction text quoted. Example inputs are shown in Figures 2-5."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Finetuning hyperparameters are reported: 'two training epochs, a learning rate of 2e-4, and a batch size of 16' with LoRA and mixed precision. However, inference parameters (temperature, top-p, max tokens) are not reported for any model, including the ChatGPT API calls."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct single-turn prompting of LLMs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3.1 describes RD dataset pruning rules (removing entries with no code intersection, eliminating meaningless comments like 'done' or 'fixed'), reducing from ~17K to 16,228. Section 3.3.2 describes PD dataset creation: 30 PMD rules selected, 20K snippets scanned, 6 developers created fixes, resulting in 14,935 entries."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'THREATS' provides a dedicated discussion of validity threats, covering model size choices and data collection biases."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The threats section is brief (one paragraph) and mostly generic. It acknowledges focusing on 6-7B models and potential data biases but does not discuss specific threats like contamination risk, Java-only generalization limits, single-run reliability, or the impact of ECM as a proxy metric."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no mention that results are limited to Java, that ECM underestimates true fix rates, or that the approach has not been validated in actual CI/CD pipelines."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The RD dataset references Tufano et al. [32] which is publicly available, but the authors' pruned version (16,228 entries) is not released. The PD dataset of 14,935 entries is not available for download."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes both datasets: RD from Tufano et al. with manual pruning rules, PD from scanning open-source Java projects on GitHub with 30 PMD rules and having 6 developers create fixes. The five components of each data instance are listed."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The six developers who created fixes for the PD dataset are described only as 'experienced developers' with no information about how they were selected, their expertise level, or potential biases. The selection of 'popular open-source Java projects' is not further specified."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented with counts: RD goes from ~17K raw to 16,228 after pruning (with rules described). PD goes from 20,000 selected snippets to 14,935 after developer review. The 9:1 train/validation split yields 28,063 training and 3,100 validation samples."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement, acknowledgments section, or grant numbers are present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: four authors from Ant Group, two from Nanjing University, and one with dual affiliation. Ant Group develops CodeFuse, one of the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Four of six authors are from Ant Group, which develops CodeFuse—one of the models evaluated. Ant Group has a commercial interest in demonstrating LLM capabilities for code repair. No independence statement is provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper. The conflict between Ant Group affiliation and CodeFuse evaluation is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models. ChatGPT-3.5/4 release dates are mentioned but not their training data cutoffs. Open-source model pre-training data temporal scope is not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Tufano et al. dataset (from GitHub/Gerrit) or the PMD dataset (from GitHub Java projects) could overlap with the pre-training data of the evaluated models, many of which were trained on GitHub code."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Tufano et al. dataset was published in 2021, before the training cutoffs of most evaluated models. ChatGPT and others may have seen these code review examples during pre-training. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The six developers who annotated the PMD dataset are data creators, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, latency measurements, or per-example timing are reported. The paper mentions models can 'infer within seconds' in the threats section but provides no specific numbers despite the stated goal of CI/CD integration."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (2x A100-80GB GPUs, 200GB RAM, AMD EPYC 7T83) and training hyperparameters are given, but total GPU hours, training time, or API costs are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds, seed sensitivity analysis, or results across multiple seeds. All results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. There is no indication of whether results are from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the same hyperparameters' were used across all models (lr=2e-4, batch=16, 2 epochs) but does not explain how these values were selected or report any search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The hyperparameters appear to be pre-selected without justification. The paper does not explain why lr=2e-4, batch=16, and 2 epochs were chosen or whether alternative configurations were explored."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors from Ant Group evaluate CodeFuse (Ant Group's model) alongside competitors. The potential bias of evaluating their own company's product is not acknowledged or addressed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 220M (CodeReviewer) to 175B (ChatGPT-3.5) parameters with vastly different compute requirements, but performance is compared without normalizing for compute budget. Only the CodeT5+ size comparison (RQ3) touches on this."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether ECM (exact code match via AST comparison) is a valid measure of repair capability. Semantically correct but syntactically different fixes are counted as failures, but this validity gap is not analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved—all evaluations use direct single-turn prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Tufano et al. dataset was published in 2021, and the GitHub code used for the PMD dataset predates the models' training. No discussion of whether these examples could be in the pre-training data of ChatGPT, CodeLLaMA, or other models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The fix range markers ([FIX_START], [FIX_END]) in P5 and P7 provide oracle information about where the fix should go. While review comments would be available in practice, the exact fix range is typically not known a priori. This potential leakage is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and validation examples share structural similarities (e.g., from the same repositories, similar PMD rule violations, or near-duplicate code patterns). The 9:1 split method is not described—random vs. repository-level splitting would affect independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, membership inference tests, decontamination pipelines, or n-gram overlap analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeLLaMA achieves a 72.97% exact-code-match repair rate with the most informative prompt (P7) combining review comments and fix range.",
       "evidence": "Table 2 shows CodeLLaMA with P7 achieves 72.97% ECM and 0.9394 Code BLEU, the highest among all model-prompt combinations (Section 4.1.2).",
       "supported": "strong"
     },
     {
       "claim": "Review comments and fix range information substantially enhance repair performance, with all models showing >20pp ECM improvement from P3 to P7.",
       "evidence": "Table 2 last row shows improvements of 21.51% to 25.10% from P3 to P7 across all six finetuned models. Table 4 breaks down the incremental contribution of each prompt component (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Fix range information provides the largest single improvement, with an average +7.69pp ECM increase from P4 to P5.",
       "evidence": "Table 4 shows the P4→P5 transition (replacing bug location with fix range) yields the highest average improvement at 7.69%, compared to 5.77% for P3→P4, 3.56% for P5→P6, and 5.77% for P6→P7 (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "6-7B parameter models represent the optimal balance between efficiency and effectiveness for this task.",
       "evidence": "Figure 8 shows ECM improvements plateau from 6B to 16B in CodeT5+. Section 4.3 reports CodeFuse-13B and LLaMA-13B show only minor gains (+2.2%, +1.87%) over 7B versions, while CodeLLaMA-13B slightly decreases (-0.29%).",
       "supported": "moderate"
     },
     {
       "claim": "Cross-dataset transferability is poor—models finetuned on one dataset lose approximately half their accuracy when tested on the other.",
       "evidence": "Table 5 shows CodeLLaMA_RD achieves 65.10% on RD_t but only 44.18% on PD_t. CodeLLaMA_PD achieves 80.65% on PD_t but only 36.31% on RD_t (Section 4.4).",
       "supported": "strong"
     },
     {
       "claim": "ChatGPT-4 outperforms ChatGPT-3.5 when review comments are provided (P2), but underperforms without them (P1).",
       "evidence": "Table 1 shows P1: ChatGPT-3.5 14.10% vs ChatGPT-4 6.61%; P2: ChatGPT-3.5 49.42% vs ChatGPT-4 55.10% (Section 4.1.1).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Four of six authors are from Ant Group, which develops CodeFuse—one of the nine evaluated models. While CodeFuse does not top the rankings (CodeLLaMA does), the potential for bias in experimental setup, hyperparameter tuning, or data selection favoring their own product is not acknowledged."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, standard deviations, or multiple-seed analysis. Given that finetuning results can vary substantially across seeds, the reported rankings may not be stable."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "The Tufano et al. dataset (from GitHub/Gerrit, published 2021) and the PMD dataset (from GitHub Java projects) could overlap with the pre-training data of ChatGPT, CodeLLaMA, and other models trained on GitHub code. This is never discussed despite being a fundamental threat to validity."
     },
     {
       "flag": "Overclaiming scope",
       "detail": "The paper tests exclusively on Java code review defects but makes general claims about 'code-review defects' and 'program repair.' The title, abstract, and conclusions do not qualify results as Java-specific."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('CodeLLaMA outperformed the others', 'ChatGPT-4 surpasses ChatGPT-3.5') are based on raw percentage comparisons without any statistical testing. Small differences (e.g., 1.48pp between P5 and P6 for CodeGen-2) may not be significant."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational LLM-for-code paper (Codex); relevant to understanding code generation model capabilities and evaluation."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Describes CodeLLaMA, the best-performing model in this study; key to understanding LLM architecture for code tasks."
     },
     {
       "title": "InferFix: End-to-End Program Repair with LLMs",
       "authors": ["Matthew Jin", "Syed Shahriar", "Michele Tufano"],
       "year": 2023,
       "arxiv_id": "2303.07263",
       "relevance": "Closely related work using LLMs for automated program repair of checker-identified bugs, with prompt engineering and Codex finetuning."
     },
     {
       "title": "CIRCLE: Continual Repair across Programming Languages",
       "authors": ["Wei Yuan", "Quanjun Zhang", "Tieke He"],
       "year": 2022,
       "doi": "10.1145/3533767.3534219",
       "relevance": "LLM-based program repair approach using prompts with context and buggy/fixed code; directly comparable approach to CR defect repair."
     },
     {
       "title": "Automating Code Review Activities by Large-Scale Pre-Training",
       "authors": ["Zhiyu Li", "Shuai Lu", "Daya Guo"],
       "year": 2022,
       "doi": "10.1145/3540250.3549081",
       "relevance": "Describes CodeReviewer, a model pre-trained specifically for code review tasks, evaluated in this paper."
     },
     {
       "title": "Towards Automating Code Review Activities",
       "authors": ["Rosalia Tufano", "Luca Pascarella", "Michele Tufano"],
       "year": 2021,
       "doi": "10.1109/ICSE43902.2021.00027",
       "relevance": "Provides the reviewer dataset (RD) used in this study; foundational work on automating code review."
     },
     {
       "title": "An Empirical Study on Learning Bug-Fixing Patches in the Wild via Neural Machine Translation",
       "authors": ["Michele Tufano", "Cody Watson", "Gabriele Bavota"],
       "year": 2019,
       "doi": "10.1145/3340544",
       "relevance": "Early neural machine translation approach to automated program repair, establishing the buggy→fixed code translation paradigm."
     },
     {
       "title": "A Survey of Learning-based Automated Program Repair",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Yuxiang Ma"],
       "year": 2023,
       "arxiv_id": "2301.03270",
       "relevance": "Comprehensive survey of learning-based APR techniques providing context for LLM-based repair approaches."
     },
     {
       "title": "LLaMA: Open and Efficient Foundation Language Models",
       "authors": ["Hugo Touvron", "Thibaut Lavril", "Gautier Izacard"],
       "year": 2023,
       "arxiv_id": "2302.13971",
       "relevance": "Describes the LLaMA model family used as a baseline; foundational open-source LLM for code and language tasks."
     },
     {
       "title": "CodeGen2: Lessons for Training LLMs on Programming and Natural Languages",
       "authors": ["Erik Nijkamp", "Hiroaki Hayashi", "Caiming Xiong"],
       "year": 2023,
       "arxiv_id": "2305.02309",
       "relevance": "Describes CodeGen-2 model evaluated in this study; insights on training LLMs for code generation tasks."
     },
     {
       "title": "Out of the BLEU: How Should We Assess Quality of the Code Generation Models?",
       "authors": ["Mikhail Evtikhiev", "Egor Bogomolov", "Yaroslav Sokolov"],
       "year": 2023,
       "doi": "10.1016/j.jss.2023.111741",
       "relevance": "Evaluates code generation metrics including Code BLEU; supports this paper's finding that Code BLEU may not be ideal for evaluating code repair."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly applicable to CI/CD pipelines for semi-automated code review defect repair, though no released tool or code limits immediate adoption."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms expected finding that more context (review comments, fix range) helps LLMs produce better fixes."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No security or safety concerns raised; the work addresses developer productivity, not AI risk."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; straightforward empirical comparison."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo is released; results cannot be independently tried."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses well-known models (ChatGPT, LLaMA) but authors are from Ant Group and Nanjing University, not top-tier ML research labs."
     }
   }
 }