ai-research-survey

scan.json (30595B)

---

 {
   "paper": {
     "title": "Enhancing Automated Program Repair through Fine-tuning and Prompt Engineering",
     "authors": [
       "Rishov Paul",
       "Md. Mohib Hossain",
       "Mohammed Latif Siddiq",
       "Masum Hasan",
       "Anindya Iqbal",
       "Joanna C. S. Santos"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2304.07840"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Fine-tuned PLBART and CodeT5 models significantly outperform prior baselines on two code review-based program repair datasets, with CodeT5 achieving up to 25.65% improvement in Top-5 accuracy on the Tufano dataset. Zero-shot and few-shot prompting with GPT-3.5-Turbo and Code-DaVinci-Edit-001 shows competitive but mixed results, with heuristic post-processing substantially improving accuracy. Manual developer analysis reveals that even the best models fulfill code review intentions in only ~53-59% of cases, indicating language models are not yet practical for automated program repair.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A Zenodo replication package is provided at https://doi.org/10.5281/zenodo.8122636, described as containing 'all the scripts used to gather the data and results.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Both datasets used (Tufano et al. and Review4Repair) are publicly available from prior work, and the replication package on Zenodo includes the processed data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using an 'NVIDIA GeForce RTX 2070-8GB GPU' but does not provide library versions, requirements.txt, Dockerfile, or other dependency specifications needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. The Zenodo replication package contains scripts, but the paper itself provides no README-like instructions or 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables II and III are reported as point estimates (e.g., '25.28%', '29.82%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims models 'outperform' baselines and reports improvements like '+10.23%' but never applies statistical significance tests (no p-values, t-tests, or other tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table II reports absolute improvements over baselines (e.g., '+5.69', '+10.23') alongside both the baseline and model absolute values, providing sufficient context to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "For RQ3 (manual analysis), the paper justifies sample sizes: 'we randomly collected 314 test samples from Tufano et al. and 340 test samples from Review4Repair datasets in order to achieve a 95% confidence interval and 5% error of margin.'"
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or results across multiple runs are reported. All results appear to be from single experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against R4R CC (Review4Repair baseline) and Tufano 2-encoder (Tufano et al. baseline), both from the respective prior works."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are from the original dataset papers (Review4Repair 2022 and Tufano et al. 2021), which are the most relevant and recent prior work on these specific datasets."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper compares different models but does not ablate components within any approach (e.g., removing pre-training, removing code review input) to isolate their contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses five metrics: Top-1 Accuracy, Top-5 Accuracy, Top-10 Accuracy (exact match at different beam sizes), BLEU-4, and CodeBLEU."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 involves two software developers manually scoring repaired code from all five models based on whether the generated fix fulfills the intention of the code review, with Cohen's Kappa for inter-rater agreement."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Both datasets use explicit train/validation/test splits. The Tufano dataset has 13,756/1,719/1,719 splits. The Review4Repair dataset was reorganized to 53,198/2,956/2,955 splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides per-category breakdowns across three fix categories (Insert, Delete, Update) for both datasets, showing each model's performance in each category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V.A discusses specific failure observations from developers: vague reviews ('nice'), context-dependent reviews ('check my previous comment'), and ground truth misalignment with focus scope. Section III-B5 details common failure patterns in LLM outputs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: few-shot prompting does not consistently outperform zero-shot (Table III, Review4Repair after heuristics), LLM accuracy is 'not satisfactory' overall, and manual analysis shows models fulfill review intentions in only ~40-59% of cases."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims pre-trained models 'notably outperformed each of the previous models' (supported by Table II) and that 'the practical application of using LLMs in the context of automated program repair is still a long way off' (supported by RQ3 results showing <60% review fulfillment)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states 'this boost in accuracy is due to mostly the learned parameters of the model rather than the architecture itself,' a causal claim about the source of improvement. No ablation separating architecture from pre-trained weights is conducted to support this. The comparison conflates model architecture, pre-training corpus, and fine-tuning effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Enhancing Automated Program Repair through Fine-tuning and Prompt Engineering' is broad, but results are limited to Java code from two code review datasets. The abstract discusses 'automated program repair' without bounding to Java or code review-based repair. The threats to validity section acknowledges the single-language limitation but the title and abstract do not."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses data leakage as an alternative explanation for GPT-3.5 performance: 'One possible reason might be that these LLMs were also trained with our aforementioned datasets.' It also discusses hyperparameter sensitivity and dataset quality issues (vague reviews) as confounding factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between automated metrics (exact match, BLEU, CodeBLEU) and actual review fulfillment (RQ3 manual analysis), acknowledging that exact match may miss valid alternative fixes. RQ3 was specifically designed to address this proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "GPT-3.5-Turbo is referenced without a version/snapshot date (e.g., 'gpt-3.5-turbo-0301'). Code-DaVinci-Edit-001 is a specific model name. PLBART and CodeT5 reference their original papers but not specific checkpoint versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Listing 2 provides the full prompt structure for both zero-shot and few-shot prompting, including the actual command text ('Refactor the Buggy Code using the Review without comments') and system role content ('You are a coding assistant. You generate only the source code.'). Fill values come from public datasets."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature=0, top_p=1, frequency_penalty=0, presence_penalty=0 for LLMs. For fine-tuning: beam sizes (1, 5, 10), epochs (11, 12, 45), batch size=4, accumulated gradient steps=8, input length=512, target length=200, patience=10."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates models via direct fine-tuning and single-turn prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A1 describes preprocessing in detail: concatenation of buggy code and review with special tokens, removal of extra spaces and newlines, handling of samples exceeding 512 tokens (57+6 removed), and reorganization of Review4Repair into 90/5/5 splits."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' is a dedicated section discussing internal and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: hyperparameter search space limitations for this specific architecture, confinement to Java as the only programming language, and potential data leakage in GPT-3.5-Turbo due to training cutoff dates relative to dataset publication."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'the datasets consisted of only Java codes and respective code reviews in the English language; hence, our focus was confined to a single programming language' and notes that 'the coverage of our findings is limited.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Both datasets are publicly available from prior work (Tufano et al. and Review4Repair), and the Zenodo replication package provides the processed data and scripts."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A1 describes both datasets: Tufano et al. contains 17,194 samples from Gerrit and GitHub, Review4Repair contains 56,211 training and 2,961 test samples from Gerrit. Preprocessing steps are detailed."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For RQ3, two developers are described as having 'one year of industry experience in a Fortune 500 company' but no details on how they were recruited or selected, or whether this introduces bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw datasets through preprocessing (tokenization, length filtering, split reorganization), model input creation, output generation, heuristic post-processing, and evaluation is documented across Sections III-A and III-B, including counts of removed samples (57+6 over 512 tokens)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: BUET (Bangladesh), University of Notre Dame, and University of Rochester. None of the authors are affiliated with companies whose products are being evaluated (OpenAI)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funders cannot be assessed. The absence of a funding statement does not confirm the work is unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure is present in the paper. Two authors list IQVIA email addresses but this affiliation is not discussed in relation to the research."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The knowledge cut-off of these two models is September 2021' for GPT-3.5-Turbo and Code-DaVinci-Edit-001 in Section VI."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses potential overlap: 'One possible reason might be that these LLMs were also trained with our aforementioned datasets. As a result, there might be a data leakage.' They note the Tufano dataset was published before September 2021 while Review4Repair was published after."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges contamination risk: the Tufano dataset was published before the models' training cutoff and could be in training data. They note 'As these models are black-box, there is no way we can verify if there is data leakage for these datasets.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the manual analysis study (RQ3)."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the developer evaluation study."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "The two developers are described as having 'one year of industry experience in a Fortune 500 company and significant involvement in the code review process in software development.'"
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for selecting the two developer evaluators. No screening process is described."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a rating/evaluation task, not an experimental study with conditions assigned to participants. Randomization of participant assignment to conditions is not applicable."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe whether the two developers were blinded to which model produced each repair. Knowing the model source could introduce bias in their ratings."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "Attrition is not explicitly reported. While results imply both raters completed all evaluations, there is no explicit statement of how many started vs. finished."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs are reported for GPT-3.5-Turbo or Code-DaVinci-Edit-001 usage, despite making API calls for all test samples across multiple prompting strategies."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using an 'NVIDIA GeForce RTX 2070-8GB GPU' but does not state GPU hours, total training time, or total API spend."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from single training runs for the fine-tuned models."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It appears each model was trained once per configuration. For API calls, temperature=0 makes output deterministic but this is not explicitly discussed as justification for single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions experimenting with 'various numbers of epochs' and different beam sizes but does not report the total number of configurations tried or the compute spent on hyperparameter search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Epoch selection is justified via validation performance: 'we used 11 epochs because we found that the model's performance remains unchanged after 11 epochs' with patience=10. CodeT5 was fine-tuned for 45 epochs 'based on observing validation losses.'"
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made (multiple models × 2 datasets × multiple metrics) but no correction for multiple comparisons is applied. No statistical tests are used at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors fine-tune their own models and compare against baseline results from other papers without acknowledging the systematic bias of evaluating one's own system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare performance relative to compute budget. Fine-tuning PLBART/CodeT5 and calling OpenAI APIs have very different compute costs, but this is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether exact match on these specific code review datasets actually measures automated program repair capability, or whether the datasets are representative of real-world repair scenarios."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated via direct fine-tuning or single-turn prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses temporal leakage: the Tufano et al. dataset was published before the models' September 2021 training cutoff, meaning solutions could be in training data. The Review4Repair dataset was published after the cutoff."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information that would not be available in real-world usage (e.g., the <START>/<END> tags marking the exact buggy location)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples share structural similarities (e.g., from the same repositories, same authors, or near-duplicate code patterns)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges leakage risk but explicitly states 'As these models are black-box, there is no way we can verify if there is data leakage' without applying any detection method (canary strings, n-gram overlap, membership inference)."
       }
     }
   },
   "claims": [
     {
       "claim": "Fine-tuned PLBART and CodeT5 significantly outperform baseline models on both code review-based APR datasets.",
       "evidence": "Table II shows CodeT5 achieves +10.23% Top-1 accuracy on Review4Repair and +21.12% on Tufano et al. PLBART achieves +5.69% and +20.82% respectively.",
       "supported": "moderate"
     },
     {
       "claim": "CodeT5 generally outperforms PLBART across most metrics and fix categories.",
       "evidence": "Table II and Figure 2 show CodeT5 achieves higher accuracy in Insert and Update categories on both datasets, and higher Top-1 accuracy overall.",
       "supported": "moderate"
     },
     {
       "claim": "The accuracy boost comes primarily from pre-trained weights rather than architecture.",
       "evidence": "The conclusion claims this but no ablation separating architecture from pre-training is conducted. The evidence is the observation that both models (different architectures) improve over baselines.",
       "supported": "weak"
     },
     {
       "claim": "Heuristic post-processing substantially improves zero-shot GPT-3.5-Turbo accuracy.",
       "evidence": "Table III shows accuracy improves from 6.9% to 22.06% on Review4Repair and from 17.86% to 30.13% on Tufano et al. after applying heuristics.",
       "supported": "strong"
     },
     {
       "claim": "Code-DaVinci-Edit-001 achieves state-of-the-art on some metrics without fine-tuning.",
       "evidence": "Table III shows it achieves the best CodeBLEU (88.42%) on Review4Repair and best accuracy (40.70%) and CodeBLEU (88.63%) on Tufano et al.",
       "supported": "moderate"
     },
     {
       "claim": "Language models are not yet capable of practically fulfilling code review intentions in automated repair.",
       "evidence": "Table IV (RQ3) shows the best models fulfill review intentions in only 52.65% (Review4Repair, CodeT5) and 58.92% (Tufano, Code-DaVinci-Edit-001) of cases.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('outperforms', 'significant improvement') are based on comparing raw accuracy numbers without any statistical significance tests, despite multiple models and datasets being compared."
     },
     {
       "flag": "Very small evaluator pool for manual analysis",
       "detail": "Only two developers evaluated the repaired code for RQ3. While Cohen's Kappa is reported (0.51-0.68, moderate agreement), two raters is minimal for a manual evaluation study. No blinding is described."
     },
     {
       "flag": "Single-run results without variance",
       "detail": "All fine-tuning results appear to be from single experimental runs with no variance or seed sensitivity analysis. Neural network training can produce substantially different results across random seeds."
     },
     {
       "flag": "Acknowledged but unmitigated data leakage risk",
       "detail": "The paper acknowledges that GPT-3.5-Turbo may have been trained on the Tufano dataset (published before its September 2021 cutoff) but does not apply any detection or mitigation method, making the GPT results potentially inflated."
     },
     {
       "flag": "Causal claim without adequate evidence",
       "detail": "The paper claims the performance boost is 'due to mostly the learned parameters of the model rather than the architecture itself' without conducting an ablation study that isolates pre-trained weights from architectural differences."
     }
   ],
   "cited_papers": [
     {
       "title": "Unified pre-training for program understanding and generation",
       "authors": ["W. Ahmad", "S. Chakraborty", "B. Ray", "K.-W. Chang"],
       "year": 2021,
       "relevance": "PLBART model used in this study; pre-trained model for code understanding and generation tasks."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Y. Wang", "W. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2021,
       "relevance": "CodeT5 model used in this study; pre-trained encoder-decoder for code tasks."
     },
     {
       "title": "Review4Repair: Code review aided automatic program repairing",
       "authors": ["F. Huq", "M. Hasan", "M. M. A. Haque", "S. Mahbub", "A. Iqbal", "T. Ahmed"],
       "year": 2022,
       "relevance": "One of the two datasets and baseline models used; demonstrates code review improves APR."
     },
     {
       "title": "Towards automating code review activities",
       "authors": ["R. Tufano", "L. Pascarella", "M. Tufano", "D. Poshyvanyk", "G. Bavota"],
       "year": 2021,
       "doi": "10.1109/ICSE43902.2021.00027",
       "relevance": "One of the two datasets and baseline models used; demonstrated transformer models for code review-based repair."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper; Code-DaVinci-Edit-001 used in this study derives from Codex."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["T. B. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "GPT-3 paper; GPT-3.5-Turbo used in this study derives from GPT-3 family."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang"],
       "year": 2020,
       "relevance": "Pre-trained code model; relevant to understanding pre-trained model capabilities for code tasks."
     },
     {
       "title": "SequenceR: Sequence-to-sequence learning for end-to-end program repair",
       "authors": ["Z. Chen", "S. Kommrusch", "M. Tufano"],
       "year": 2021,
       "relevance": "Sequence-to-sequence approach for automated program repair; directly relevant to APR methodology."
     },
     {
       "title": "CoCoNut: Combining context-aware neural translation models using ensemble for program repair",
       "authors": ["T. Lutellier", "V. H. Pham", "L. Pang", "Y. Li", "M. Wei", "L. Tan"],
       "year": 2020,
       "relevance": "Neural machine translation approach for APR using ensemble methods."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan", "B. Dolan-Gavitt", "R. Karri"],
       "year": 2022,
       "doi": "10.1109/SP46214.2022.00057",
       "relevance": "Security evaluation of LLM-generated code; relevant to understanding quality and safety of AI code generation."
     },
     {
       "title": "GitHub Copilot AI pair programmer: Asset or liability?",
       "authors": ["A. Moradi Dakhel", "V. Majdinasab", "A. Nikanjam", "F. Khomh"],
       "year": 2023,
       "relevance": "Evaluates correctness of Copilot-generated code; relevant to LLM code generation quality assessment."
     },
     {
       "title": "Exploring the effectiveness of large language models in generating unit tests",
       "authors": ["M. L. Siddiq", "J. C. S. Santos", "R. H. Tanvir"],
       "year": 2023,
       "relevance": "Uses zero-shot prompting with GPT-3.5-Turbo for code generation tasks; directly relevant methodology."
     },
     {
       "title": "Extracting training data from large language models",
       "authors": ["N. Carlini", "F. Tramèr", "E. Wallace"],
       "year": 2020,
       "relevance": "Data leakage and training data extraction from LLMs; relevant to contamination concerns in LLM evaluation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides concrete fine-tuning and prompt engineering recipes for code repair with publicly available tools and datasets."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that LLMs still fail to fulfill review intentions ~50% of the time is mildly surprising given the hype around LLMs for code tasks."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with established claims."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Zenodo replication package with scripts is available, but no live demo or pip-installable tool."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses GPT-3.5-Turbo (recognizable) but the authors and institutions are not widely known."
     }
   }
 }