ai-research-survey

scan.json (30853B)

---

 {
   "paper": {
     "title": "Harnessing Large Language Models for Curated Code Reviews",
     "authors": [
       "Oussama Ben Sghaier",
       "Martin Weyssow",
       "Houari Sahraoui"
     ],
     "year": 2025,
     "venue": "IEEE Working Conference on Mining Software Repositories",
     "arxiv_id": "2502.03425",
     "doi": "10.1109/MSR66628.2025.00039"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Replication package released at https://github.com/OussamaSghaier/CuREV, mentioned in footnote and Data Availability section: 'We publicly release all the code, models, data, and results of our experiments.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on Zenodo at https://zenodo.org/records/14812107, explicitly stated in the abstract footnote and Data Availability section."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (four NVIDIA RTX A5000 GPUs, 24GB each) and LoRA settings, but provides no requirements.txt, Dockerfile, or library version specifications sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references a replication package but does not include step-by-step reproduction instructions in the paper itself. No README contents, commands, or scripts to replicate experiments are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates only (BLEU 7.71 vs 11.26, CodeBLEU 0.36 vs 0.44, EM 408 vs 445). No confidence intervals, error bars, or ± notation anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (46% BLEU, 22% CodeBLEU) based solely on comparing two numbers. No statistical significance tests (t-test, bootstrap, etc.) are applied to any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: '46% improvement in BLEU score' (7.71→11.26), '22% improvement in CodeBLEU' (0.36→0.44), and absolute differences for scoring criteria (clarity 6.89→8.96, conciseness 7.71→8.05)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The subset of 20,000 comments used for downstream experiments is not justified. No power analysis or explanation of why 20,000 was chosen from the 170,718 available samples."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run numbers. No standard deviations, variance across seeds, or spread measures are reported for any experiment."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The original (uncurated) dataset serves as the baseline for all comparisons. Models trained on original vs curated data are compared directly under identical conditions."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "DeepSeek-Coder-6.7B-Instruct (2024) and Llama-3.1-70B (2024) are contemporary models. The baseline comparison is between dataset versions rather than competing methods, making recency of competing approaches less relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The curation pipeline has two components — filtering irrelevant comments and reformulating comments — but no ablation study separates their individual contributions to the improvements."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: BLEU for comment generation (RQ3); CodeBLEU and Exact Match for code refinement (RQ4); clarity, conciseness, and relevance scores for dataset quality (RQ1/RQ2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "A sanity check involving two authors manually evaluating 100 review comments was conducted (Section III-D). Cohen's kappa was computed between human and LLM judgments across all categories and criteria, validating the LLM-as-judge outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A states: 'We further split each subset into 75% for training and 25% for evaluation.' The evaluation set is separate from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table V provides scoring criteria broken down by type (refactoring, bugfix, testing, etc.), nature (descriptive, prescriptive, clarification), and civility subcategories. Table I shows distribution across 9 programming languages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table XI shows an example where the model trained on original data generates an incorrect review comment. Examples of filtered low-quality comments are shown in Section IV-A. The modest conciseness improvement is attributed to LLM verbosity."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges the conciseness improvement was modest (only +0.34 vs +2.07 for clarity), attributed to 'the inherently verbose nature of LLMs, which tend to produce more elaborate text by default.' The Other/Nature category also increased unexpectedly from 0.01% to 9.90%."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '46% improvement in BLEU score' (7.71→11.26 = 46%) and '22% improvement in CodeBLEU' (0.36→0.44 = 22%) match Tables X and XII. Claims about clarity and conciseness improvements match Table VII."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims curation 'leads to enhanced model performance.' The experimental design is a controlled comparison: same model, same hyperparameters, same paired samples, with only the dataset version varying. This single-variable manipulation is adequate for the causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only one downstream model (DeepSeek-6.7B-Instruct) and one curation LLM (Llama-3.1-70B) but claims generalized improvements for 'LLMs' broadly. The abstract says 'enhanced model performance' and conclusions refer to 'the effectiveness of LLMs for code review tasks' without bounding to the specific model tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity (Section VI) discusses data noise and LLM-as-judge reliability but does not consider alternative explanations for the results. For example, curated comments could be easier to learn due to reduced variance (formulaic phrasing) rather than higher quality. No robustness checks against this."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures BLEU and frames it as 'accuracy' of comment generation. BLEU measures n-gram overlap with reference text, not actual comment quality or usefulness. This gap between the proxy metric and the claimed outcome (better comments) is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'Llama-3.1-70B-Instruct' and 'DeepSeek-Coder-6.7B-Instruct' (with citation to the DeepSeek-Coder paper). These include version numbers and size specifications."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompt excerpts are provided in Tables IV and VI. The paper states 'The full prompt is available in the replication package' with a GitHub URL. While only excerpts appear in the paper, the replication package contains the complete prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Fine-tuning hyperparameters are reported (batch size 4, 5 epochs, LoRA r=16, α=32, dropout=0.05) but inference settings for Llama-3.1-70B (temperature, top-p, max tokens for evaluation/reformulation of 176K samples) and DeepSeek inference (temperature for code refinement) are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct LLM prompting and fine-tuning without agent loops, tool use, or scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The filtering step is documented with specific criteria: relevance threshold of 4, resulting in 5,895 samples removed (176,613→170,718). The reformulation process using Llama-3.1-70B is described with the prompt guidelines. The 20,000-sample subset selection for downstream experiments is described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' is a dedicated section discussing potential limitations of the work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses specific issues: (1) non-English or misspelled words in review comments, mitigated by BPE tokenization, and (2) LLM-as-judge reliability, validated by sanity check on 100 samples with Cohen's kappa scores. These are specific to this study's design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No mention of limitations regarding the single downstream model tested, the single curation LLM, or specific settings/populations excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Both the original dataset (from Li et al., publicly available) and the curated CuRev dataset are released on Zenodo (https://zenodo.org/records/14812107). The Data Availability section confirms: 'We publicly release all the code, models, data, and results.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the source dataset (Li et al., largest publicly available code review dataset), its composition (176,613 samples, 9 programming languages), and content (code diffs + review comments from pull requests). Table I shows the distribution across languages."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The data source is a standard publicly available benchmark dataset (code reviews from repositories). The two-author sanity check is a validation step, not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: original dataset (176,613) → LLM evaluation → relevance filtering at threshold 4 (removing 5,895 samples) → curated dataset (170,718) → reformulation with Llama-3.1-70B → re-evaluation. For downstream tasks: 20,000 paired samples → 75/25 train/test split."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers appear anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Université de Montréal (Sghaier, Sahraoui) and Singapore Management University (Weyssow). No commercial product is being evaluated, so no conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. This defaults to NO since the absence of disclosure is not the same as absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Neither Llama-3.1-70B nor DeepSeek-Coder-6.7B-Instruct training data cutoff dates are stated. The models could have been trained on the Li et al. code review dataset."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the code review dataset (publicly available since 2022) appeared in the pre-training data of DeepSeek or Llama models. This is especially concerning for the code refinement task (RQ4) where DeepSeek is used directly for inference."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Li et al. code review dataset was published in 2022, before both Llama-3.1 and DeepSeek-Coder were trained. The models could have seen the test examples during pre-training. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. The two-author sanity check on 100 samples is an inter-rater reliability validation, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. The work analyzes publicly available code review data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, tokens consumed, or wall-clock time reported for running Llama-3.1-70B on 176,613 samples or for DeepSeek inference on 5,000 test samples."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (four NVIDIA RTX A5000 GPUs, 24GB each) but total training time, GPU hours, and compute budget for the full pipeline (evaluation + curation + fine-tuning + inference) are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from a single run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating whether they are from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "LoRA settings (r=16, α=32, dropout=0.05) are reported but no search budget, search method, or number of configurations tried is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selected LoRA configuration and other hyperparameters are presented without explanation of how they were chosen or whether any selection process was used."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "Only a small number of comparisons are made (original vs curated on two tasks), and no statistical tests are applied in the first place."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors create the curation pipeline and evaluate it themselves. They do not acknowledge the bias of evaluating their own system. Furthermore, the same LLM (Llama-3.1-70B) that performs the curation also performs the re-evaluation, creating circular evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance curves across compute levels. The curation pipeline requires running Llama-3.1-70B on 176K samples, which is substantial compute, but no comparison of cost vs benefit is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether BLEU and CodeBLEU actually measure comment quality and code refinement quality. BLEU has known weaknesses for evaluating natural language generation quality, but this is not acknowledged."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; the approach uses direct prompting and fine-tuning."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the code review dataset (published 2022) was included in the training data of Llama-3.1 or DeepSeek-Coder models, both trained after 2022."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For code refinement (RQ4), providing the review comment could leak information about the expected code change."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples are independent. Samples from the same repository or same pull request could share structural patterns, inflating performance."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods (canary strings, membership inference, n-gram overlap analysis, decontamination) are applied."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "claims": [
     {
       "claim": "85.9% of code review samples are related to refactoring and bugfix, with 62.6% being prescriptive comments.",
       "evidence": "Section III-E reports categorization results: Refactoring 80.07%, Bugfix 18.60% (multi-labeled), Prescriptive 62.6%. Figure 3 visualizes distributions.",
       "supported": "strong"
     },
     {
       "claim": "The curation pipeline improves clarity from 6.89 to 8.96 and conciseness from 7.71 to 8.05 on average.",
       "evidence": "Table VII shows scoring criteria evolution across all categories. Average clarity improved by +2.07 and conciseness by +0.34. Figure 5 shows the curated distribution.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning on curated data yields a 46% improvement in BLEU score for comment generation (7.71→11.26).",
       "evidence": "Table X compares BLEU scores for DeepSeek-6.7B-Instruct trained on original (7.71) vs curated (11.26) datasets. Section V-A describes the experimental setup.",
       "supported": "weak"
     },
     {
       "claim": "Curated comments lead to 22% improvement in CodeBLEU (0.36→0.44) and higher Exact Match (408→445) for code refinement.",
       "evidence": "Table XII reports CodeBLEU and EM scores for original vs curated review comments used as input to DeepSeek-6.7B-Instruct. Section V-B describes the setup.",
       "supported": "weak"
     },
     {
       "claim": "LLM-as-a-Judge (Llama-3.1-70B) achieves near-perfect to substantial agreement with human evaluators on the evaluation framework.",
       "evidence": "Section III-D reports Cohen's kappa: civility 1.0, type 0.88, nature 0.82, relevance 0.85, conciseness 0.76, clarity 0.64, based on 100 manually evaluated samples.",
       "supported": "moderate"
     },
     {
       "claim": "The curation pipeline eliminates all uncivil comments (98.77%→100% civil) and shifts comments toward prescriptive nature (62.6%→90.2%).",
       "evidence": "Table VIII shows curated dataset category statistics: 100% civil, 90.20% prescriptive. Compared to original dataset figures in Section III-E.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "This paper proposes CuRev, an LLM-driven curation pipeline for code review datasets that filters irrelevant comments and reformulates remaining ones for clarity, conciseness, and civility using Llama-3.1-70B. Applied to the largest public code review dataset (176K samples), the pipeline improved clarity scores from 6.89 to 8.96 and achieved 100% civil comments. Fine-tuning DeepSeek-6.7B-Instruct on curated data yielded BLEU 11.26 vs 7.71 for comment generation, and CodeBLEU 0.44 vs 0.36 for code refinement, though these improvements lack statistical significance testing and are based on a single model evaluation.",
   "red_flags": [
     {
       "flag": "Circular evaluation — same LLM curates and judges",
       "detail": "Llama-3.1-70B is used both to reformulate review comments AND to re-evaluate the curated dataset using the same scoring criteria. The LLM is effectively judging its own output, likely inflating improvement metrics. The paper acknowledges this model is used for both tasks but does not discuss the circularity."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (46% BLEU improvement, 22% CodeBLEU improvement) are based on comparing two point estimates from apparently single runs. No significance tests, confidence intervals, or variance measures are provided. The differences could be within normal random variation."
     },
     {
       "flag": "Very low absolute BLEU scores",
       "detail": "BLEU scores of 7.71 and 11.26 are extremely low in absolute terms, suggesting neither model is generating high-quality comments. A 46% relative improvement from a very low baseline may not reflect meaningful practical improvement."
     },
     {
       "flag": "Single downstream model tested",
       "detail": "Only DeepSeek-6.7B-Instruct is used for downstream evaluation. Claims about curation improving 'LLM performance' generally are not supported by testing on a single model. The effect could be model-specific."
     },
     {
       "flag": "No ablation of pipeline components",
       "detail": "The curation pipeline has two distinct steps — filtering and reformulation — but their individual contributions are never separated. The filtering step removes 5,895 samples while reformulation changes all remaining 170,718. It's unclear which step drives the downstream improvements."
     },
     {
       "flag": "Tiny human validation sample",
       "detail": "The LLM-as-judge sanity check uses only 100 samples out of 176,613 (0.057%), and clarity agreement was only 'substantial' (kappa=0.64), the lowest among all categories. This is a thin basis for trusting LLM-generated scores across the entire dataset."
     }
   ],
   "cited_papers": [
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Z. Li", "S. Lu", "D. Guo"],
       "year": 2022,
       "relevance": "Introduced the largest public code review dataset (176K samples) used as the base dataset in this study; core work on LLM pre-training for code review automation."
     },
     {
       "title": "Towards automating code review activities",
       "authors": ["R. Tufano", "L. Pascarella", "M. Tufano", "D. Poshyvanyk", "G. Bavota"],
       "year": 2021,
       "relevance": "Early work on using T5 transformers for automated review comment generation, establishing the comment generation task evaluated in this paper."
     },
     {
       "title": "Improving the learning of code review successive tasks with cross-task knowledge distillation",
       "authors": ["O. Ben Sghaier", "H. Sahraoui"],
       "year": 2024,
       "relevance": "Prior work by the same first author on cross-task knowledge distillation for code review, directly building toward this curation approach."
     },
     {
       "title": "Llama-reviewer: Advancing code review automation with large language models through parameter-efficient fine-tuning",
       "authors": ["J. Lu", "L. Yu", "X. Li", "L. Yang", "C. Zuo"],
       "year": 2023,
       "relevance": "Demonstrates LoRA-based fine-tuning of LLMs for code review, the same technique used in this paper's downstream experiments."
     },
     {
       "title": "DeepSeek-coder: When the large language model meets programming",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "The code LLM fine-tuned and used for inference in this paper's downstream evaluation experiments."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis", "Z. Allen-Zhu"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning technique used in this paper to train DeepSeek on original and curated datasets."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-bench and chatbot arena",
       "authors": ["L. Zheng", "W.-L. Chiang", "Y. Sheng"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-judge methodology that this paper relies on for automated evaluation of code review comments."
     },
     {
       "title": "CodeUltraFeedback: An LLM-as-a-judge dataset for aligning large language models to coding preferences",
       "authors": ["M. Weyssow", "A. Kamanda", "H. Sahraoui"],
       "year": 2024,
       "arxiv_id": "2403.09032",
       "relevance": "LLM-as-judge approach for code-related tasks by co-authors; directly informed the evaluation methodology used in this paper."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu"],
       "year": 2023,
       "relevance": "Comprehensive survey of LLMs for software engineering tasks including code review, providing context for this work."
     },
     {
       "title": "Code review automation: strengths and weaknesses of the state of the art",
       "authors": ["R. Tufano", "O. Dabić", "A. Mastropaolo", "M. Ciniselli", "G. Bavota"],
       "year": 2024,
       "relevance": "Analysis of code review automation approaches including categorization of review types, which informed this paper's evaluation framework."
     },
     {
       "title": "Exploring parameter-efficient fine-tuning techniques for code generation with large language models",
       "authors": ["M. Weyssow", "X. Zhou", "K. Kim", "D. Lo", "H. Sahraoui"],
       "year": 2023,
       "arxiv_id": "2308.10462",
       "relevance": "Study of parameter-efficient fine-tuning for code generation by co-authors, informing the LoRA-based approach used here."
     },
     {
       "title": "RepairLLaMA: Efficient representations and fine-tuned adapters for program repair",
       "authors": ["A. Silva", "S. Fang", "M. Monperrus"],
       "year": 2023,
       "arxiv_id": "2312.15698",
       "relevance": "LoRA-based fine-tuning for automated program repair, showing parameter-efficient fine-tuning effectiveness for code-related tasks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The curation pipeline could be applied to other noisy code review datasets, and the curated dataset is publicly released for direct use."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "The finding that cleaner training data improves model performance confirms conventional wisdom rather than challenging it."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns are raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or confrontational claims; straightforward dataset curation paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub and data on Zenodo; a practitioner could replicate the pipeline, though no live demo or pip-installable tool."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "University researchers from Université de Montréal and Singapore Management University; no famous lab or product association."
     }
   }
 }