ai-research-survey

scan.json (25563B)

---

 {
   "paper": {
     "title": "Improving Automated Secure Code Reviews: A Synthetic Dataset for Code Vulnerability Flaws",
     "authors": [
       "Leonardo Centellas-Claros",
       "Juan J. Alonso-Lecaros",
       "Juan Pablo Sandoval Alcocer",
       "Andres Neyem"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2504.16310",
     "doi": "10.48550/arXiv.2504.16310"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["theoretical"],
   "key_findings": "This paper proposes a methodology for generating synthetic vulnerability-focused code review datasets using LLMs, but presents no completed experiments. Early data collection identified 5,973 Java repositories from GitHub and mined 3,827,517 commits, filtering to 35,950 potentially security-related commits via keyword matching. The authors plan to evaluate GPT-4o, Claude 3.5 Sonnet, Flan-T5, and Qwen 2.5 with three prompting strategies (zero-shot, CoT, self-reflection) and fine-tune CodeReviewer models, but none of this work has been executed.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code, scripts, or repository URLs are provided in the paper. The data collection pipeline and planned experiments have no associated code release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The synthetic dataset is the proposed contribution but has not been created or released. The filtered commits (35,950) are not made available. No dataset download links are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or setup instructions are provided. The paper does not describe any technical environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are included. The methodology section describes planned steps but provides no scripts, commands, or concrete instructions for reproducing the work."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents no experimental results — only descriptive counts from data collection (5,973 repos, 3.8M commits, 35,950 filtered). No quantitative results require uncertainty measures."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative claims are made based on data. The paper is a proposal with no experimental results to test for significance."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No effects have been measured. The paper is a research proposal with no completed experiments."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a proposal/theoretical paper with no completed experiments. The planned sample of 100 commits for evaluation is justified contextually ('to make manual review feasible') but no power analysis or formal justification is provided."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs have been conducted. The paper is a research proposal with no results to report variance for."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes planned baselines (CodeReviewer, Lin et al.'s version, GPT-4o) in Section III.C.1, but no baseline comparisons have been executed. Only plans are presented."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Planned baselines include CodeReviewer (2022) and Lin et al. (2024), which are reasonably contemporary. However, since no comparisons have been run, this criterion is not satisfied."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (4 LLMs × 3 prompting strategies). The planned comparison across these combinations could serve as an ablation, but no experiments have been conducted."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper plans to use BLEU-4 and manual evaluation (semantic equivalence, applicability) per Section III.C.4, but no metrics have been computed since no experiments were run."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluation by two authors is planned for both RQ1 (1,200 reviews) and RQ2 (manual assessment), with Cohen's Kappa for inter-rater agreement. However, none of this evaluation has been conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper plans to use a filtered subset of Li et al.'s test partition (Section III.C.3), identifying 43 and 63 potentially security-related samples. This test set has been preliminarily identified but not used in any evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No results are presented, so no per-category breakdowns exist. The planned evaluation across 4 LLMs and 3 prompt strategies would provide breakdowns, but this work has not been done."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases are discussed since no experiments have been conducted. The paper only presents data collection counts."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results of any kind (positive or negative) are reported. The paper is entirely a proposal."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'We anticipate that the synthetic dataset will improve the performance of the original code review models.' This anticipatory claim has no supporting evidence — no experiments have been run to test this hypothesis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the synthetic dataset will 'improve the performance' of code review models, which is a causal claim. No experimental evidence supports this, and no causal identification strategy is described beyond a planned before/after comparison."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Improving Automated Secure Code Reviews: A Synthetic Dataset for Code Vulnerability Flaws' and the abstract frame the approach as general, mentioning no language restriction. The Java-only scope is revealed only in Section III.B.1, and the title does not reflect this substantial limitation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed since no results exist. The threats to validity section (Section V) discusses methodological limitations but does not consider alternative explanations for any findings."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper plans to use BLEU-4 as a primary metric for code review quality. BLEU is a well-known imperfect proxy for text generation quality, and no discussion is provided of the gap between BLEU scores and actual usefulness of security code reviews."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists 'GPT-4o from OpenAI', 'Claude 3.5 Sonnet from Anthropic', 'Flan-T5 from Google' and 'Qwen 2.5' (Section III.B.4). These are marketing names without specific version identifiers, snapshot dates, or model sizes (especially for Flan-T5 and Qwen 2.5)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "One zero-shot prompt template is provided in Section III.B.5 with {{Diff}} and {{Message}} placeholders. However, the CoT and Self-Reflection prompts are only described in natural language ('a technique designed to enhance the model's performance'), and the actual prompt texts are not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for any of the planned LLM experiments (temperature, top-p, max tokens). For the planned fine-tuning, the paper states they will 'use the same hyperparameters as those employed in the original CodeReviewer fine-tuning tasks' without listing them."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The proposed approach involves direct prompting of LLMs without tool use, memory, or multi-step agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data preprocessing pipeline is well-documented across Sections III.B.1-3 and IV: project selection (Java, ≥50 PRs), commit filtering (single .java file, no merge commits, no non-source changes), keyword-based vulnerability filtering from Alfadel et al. [23], and test file exclusion. Counts at each stage: 5,973 repos → 3,827,517 commits → 43,131 keyword-filtered → 35,950 after test exclusion."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Threats to Validity' is a dedicated section discussing both internal and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: subjective evaluation bias from author-based review (mitigated by two independent reviewers), keyword-based filtering potentially missing security commits, prompt refinement sample of 50 commits potentially unrepresentative, and Java-only limitation. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V explicitly states: 'The study's focus on Java projects inherently limits the generalizability of its findings to other programming languages' and 'the study targets vulnerability-fixing commits—a specific subset of code review scenarios.' These are specific scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is released. The 35,950 filtered commits, keyword lists, or repository identifiers are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail in Sections III.B.1-3: Java projects from GitHub with ≥50 PRs (Tufano et al.'s criteria), single-file commits excluding merge commits and non-source files, keyword-based vulnerability filtering with iterative refinement by two authors."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Repository selection criteria are described: GitHub Java projects with at least 50 pull requests, following criteria from Tufano et al. [13]. The selection method and its rationale ('targeting repositories with significant development activity') are documented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented with counts at each stage in Section IV: 5,973 repos → 3,827,517 commits → 43,131 keyword-filtered → 35,950 after test file exclusion. Filtering criteria are described for each step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses funding: ANID Doctoral Scholarship (2024-21240734), CENIA (BASAL FB210017), and Programa de Inserción Académica 2022 at PUC Chile."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as affiliated with the Department of Computer Science, Pontificia Universidad Católica de Chile. No product from the authors' institution is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from Chilean national research agencies (ANID) and an academic AI center (CENIA). These funders have no commercial interest in the outcome of this code review research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Absence of disclosure is not absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation on any benchmark has been conducted. The paper is a proposal; the LLMs are planned for use as synthetic data generators, and no evaluation results are presented."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation has been conducted. The paper proposes future experiments but has not executed them, so train/test overlap is not yet a concern to address."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation has been performed. The paper is a research proposal with only data collection counts reported."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The planned evaluations involve the paper's authors reviewing generated outputs, not external human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study mines public GitHub repositories and plans author-based evaluation of generated text."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study with conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a proposal/theoretical paper with no completed experiments. No inference or API costs are applicable since no models have been run."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a proposal paper with no completed experiments requiring compute budget reporting."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs can produce human-like code reviews from vulnerability-related commits with high accuracy, creating a valuable resource for enhancing automated code-to-comment review models.",
       "evidence": "This is the paper's overarching hypothesis stated in Section III. No experimental evidence is provided — the claim is entirely aspirational.",
       "supported": "unsupported"
     },
     {
       "claim": "The synthetic dataset will improve the performance of existing code review models when used for fine-tuning.",
       "evidence": "Abstract states 'We anticipate that the synthetic dataset will improve the performance of the original code review models.' No experiments have been conducted to test this.",
       "supported": "unsupported"
     },
     {
       "claim": "Security-focused reviews comprise less than 4% of existing code review datasets.",
       "evidence": "Section I cites Yu et al. [8] who found only 614 security-related comments out of 20,000 code review comments (3.07%). This is based on cited literature, not the authors' own analysis.",
       "supported": "moderate"
     },
     {
       "claim": "35,950 potentially security-related commits were identified from 5,973 Java repositories on GitHub.",
       "evidence": "Section IV reports: 5,973 repositories → 3,827,517 commits → 43,131 keyword-filtered → 35,950 after test file exclusion. These are descriptive counts from the data collection pipeline.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Proposal paper with no completed experiments",
       "detail": "The paper presents no experimental results — only a planned methodology and early data collection counts. All research questions (RQ1, RQ2) remain unanswered. This is a research proposal, not a completed study."
     },
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The abstract claims 'We anticipate that the synthetic dataset will improve the performance of the original code review models' despite having zero experimental evidence. The entire evaluation framework is planned but unexecuted."
     },
     {
       "flag": "Title and abstract overgeneralize",
       "detail": "The title 'Improving Automated Secure Code Reviews: A Synthetic Dataset for Code Vulnerability Flaws' and abstract present the approach as general, but the study is restricted to Java-only projects. This limitation is only revealed in Section III.B.1."
     },
     {
       "flag": "Unvalidated filtering methodology",
       "detail": "The keyword-based commit filtering (yielding 35,950 commits) has not been validated. The planned iterative refinement with two independent reviewers and Cohen's Kappa measurement has not been executed, so the precision of the current keyword list is unknown."
     }
   ],
   "cited_papers": [
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Z. Li", "S. Lu", "D. Guo", "N. Duan", "S. Jannu", "G. Jenks", "D. Majumder", "J. Green", "A. Svyatkovskiy", "S. Fu", "N. Sundaresan"],
       "year": 2022,
       "doi": "10.1145/3540250.3549081",
       "relevance": "Introduces CodeReviewer, a large-scale pre-trained model for code review automation, used as a baseline in this work."
     },
     {
       "title": "Auger: automatically generating review comments with pre-training models",
       "authors": ["L. Li", "L. Yang", "H. Jiang", "J. Yan", "T. Luo", "Z. Hua", "G. Liang", "C. Zuo"],
       "year": 2022,
       "doi": "10.1145/3540250.3549099",
       "relevance": "Proposes automated review comment generation using GitHub data and data augmentation, directly related to code review automation."
     },
     {
       "title": "LLaMA-Reviewer: Advancing Code Review Automation with Large Language Models through Parameter-Efficient Fine-Tuning",
       "authors": ["J. Lu", "L. Yu", "X. Li", "L. Yang", "C. Zuo"],
       "year": 2023,
       "doi": "10.1109/ISSRE59848.2023.00026",
       "relevance": "Demonstrates parameter-efficient fine-tuning of LLMs for code review, relevant to LLM-based code review automation."
     },
     {
       "title": "D-act: Towards diff-aware code transformation for code review under a time-wise evaluation",
       "authors": ["C. Pornprasit", "C. Tantithamthavorn", "P. Thongtanunam", "C. Chen"],
       "year": 2023,
       "relevance": "Proposes diff-aware code transformation for automated code review, contributing to the code review automation literature."
     },
     {
       "title": "Code review automation: Strengths and weaknesses of the state of the art",
       "authors": ["R. Tufano", "O. Dabić", "A. Mastropaolo", "M. Ciniselli", "G. Bavota"],
       "year": 2024,
       "doi": "10.1109/TSE.2023.3348172",
       "relevance": "Evaluates state-of-the-art code review tools, finding up to 25% training data noise and poor performance on specific tasks."
     },
     {
       "title": "Improving automated code reviews: Learning from experience",
       "authors": ["H. Y. Lin", "P. Thongtanunam", "C. Treude", "W. Charoenwet"],
       "year": 2024,
       "doi": "10.1145/3643991.3644910",
       "relevance": "Proposes improvements to CodeReviewer through better training data, used as a baseline model in this work."
     },
     {
       "title": "Using pre-trained models to boost code review automation",
       "authors": ["R. Tufano", "S. Masiero", "A. Mastropaolo", "L. Pascarella", "D. Poshyvanyk", "G. Bavota"],
       "year": 2022,
       "doi": "10.1145/3510003.3510621",
       "relevance": "Demonstrates pre-training techniques for code review models with 1.4M samples, foundational work in automated code review."
     },
     {
       "title": "Security Defect Detection via Code Review: A Study of the OpenStack and Qt Communities",
       "authors": ["J. Yu", "L. Fu", "P. Liang", "A. Tahir", "M. Shahin"],
       "year": 2023,
       "doi": "10.1109/ESEM56168.2023.10304852",
       "relevance": "Quantifies the scarcity of security-related code reviews (614/20,000), directly motivating the need for security-focused review datasets."
     },
     {
       "title": "Exploring the potential of ChatGPT in automated code refinement: An empirical study",
       "authors": ["Q. Guo", "J. Cao", "X. Xie", "S. Liu", "X. Li", "B. Chen", "X. Peng"],
       "year": 2024,
       "doi": "10.1145/3597503.3623306",
       "relevance": "Empirical study of ChatGPT for code refinement, relevant to LLM capabilities in code review tasks."
     },
     {
       "title": "Synthetic dialogue dataset generation using LLM agents",
       "authors": ["Y. Abdullin", "D. Molla-Aliod", "B. Ofoghi", "J. Yearwood", "Q. Li"],
       "year": 2024,
       "arxiv_id": "2401.17461",
       "relevance": "Demonstrates LLM-based synthetic dataset generation via agent dialogues, directly relevant to the synthetic data generation methodology proposed."
     },
     {
       "title": "Unnatural instructions: Tuning language models with (almost) no human labor",
       "authors": ["O. Honovich", "T. Scialom", "O. Levy", "T. Schick"],
       "year": 2023,
       "relevance": "Shows effectiveness of training models on synthetic instruction data, supporting the viability of synthetic dataset approaches."
     },
     {
       "title": "ZeroGen: Efficient zero-shot learning via dataset generation",
       "authors": ["J. Ye", "J. Gao", "Q. Li", "H. Xu", "J. Feng", "Z. Wu", "T. Yu", "L. Kong"],
       "year": 2022,
       "arxiv_id": "2202.07922",
       "relevance": "Introduces LLM-based dataset generation for zero-shot learning with task-specific prompts, relevant to synthetic data generation methodology."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Proposes a method to generate security-focused review data, potentially useful for practitioners building code review tools, but nothing is released or available yet."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the well-known observation that security reviews are underrepresented in datasets and that LLMs can generate synthetic data — neither claim is surprising."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Touches on code security vulnerabilities but does not demonstrate novel attacks or raise new safety concerns beyond the known gap in security review coverage."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenges to existing work, no adversarial framing."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, dataset, model, or demo is released. The paper is purely a research proposal."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Pontificia Universidad Católica de Chile, not a widely recognized AI lab. No famous products or models are the focus."
     }
   }
 }