ai-research-survey

scan.json (29206B)

---

 {
   "paper": {
     "title": "A Multi-Dataset Evaluation of Models for Automated Vulnerability Repair",
     "authors": [
       "Zanis Ali Khan",
       "Aayush Garg",
       "Qiang Tang"
     ],
     "year": 2025,
     "venue": "ARES",
     "arxiv_id": "2506.04987",
     "doi": "10.48550/arXiv.2506.04987"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeT5 generally outperforms CodeBERT on complex vulnerability datasets (e.g., Vul4J CodeBLEU 0.94 vs 0.37) while CodeBERT handles fragmented or incomplete code better (Go, PHP). Fine-tuning improves in-distribution performance but both models struggle significantly on out-of-distribution data, with same-language cross-dataset transfer performing better than cross-language. Only BLEU-based similarity metrics were used—no functional correctness evaluation of generated patches.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 (Data Availability): 'the source code and datasets used in our study are publicly available on Zenodo [24].' Reference [24] provides DOI 10.5281/zenodo.15599983."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All six datasets are publicly available: Go/PHP on Zenodo, MegaVul on GitHub, Vul4J on GitHub, CodeParrot on HuggingFace. The authors also released their processed data via Zenodo [24]."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 mentions 'PyTorch 2.0.1 framework with CUDA 12 compatibility' and hardware specs (Xeon Silver 4210, V100 GPUs), but no requirements.txt, Dockerfile, or comprehensive dependency list is provided in the paper. The Python version is not stated."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The Zenodo artifact is referenced but no README contents or reproduction steps are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports only point estimates (e.g., CodeBLEU 0.7641, CrystalBLEU 0.6557) with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims that 'CodeT5 outperforms CodeBERT' are based solely on comparing raw score values without any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes are reported. Results are presented as raw scores in Table 2 without Cohen's d, relative improvement calculations with baseline context, or other structured magnitude measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for dataset sizes or power analysis. The choice of 85/15 split is justified only by citation to a computer vision paper [42], not by any analysis specific to this domain."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 acknowledges 'non-deterministic behavior (e.g., small variations in accuracy)' but no standard deviations, variance, or spread measures across runs are reported anywhere."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "CodeBERT and CodeT5 serve as mutual baselines, compared against each other across all six datasets in Table 2. However, no external baselines from prior vulnerability repair work (e.g., VRepair, VulRepair) are included."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "CodeBERT (2020) and CodeT5 (2021) are both 4-5 years old at the time of publication. More recent code LLMs (CodeT5+, StarCoder, DeepSeek-Coder, CodeLlama) and vulnerability-specific repair tools are not compared."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The preprocessing pipeline has multiple components (token filtering, comment removal, normalization) whose individual contributions are not measured."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: CodeBLEU and CrystalBLEU, as described in Section 4.3 and reported in Table 2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated patches is performed. All evaluation is automated via CodeBLEU and CrystalBLEU similarity metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2: 'The datasets were partitioned into 85% for training and 15% for testing... all overlapping or duplicate instances were excluded.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-dataset results for both models and both metrics. Figures 1 and 2 show cross-dataset heatmaps for the out-of-distribution evaluation."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses why models fail on fragmented/incomplete code (Go, PHP datasets). Section 5.2 discusses the performance drop on out-of-distribution data. CodeBERT's poor Vul4J performance is analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that both models struggle with OOD generalization, that fine-tuning doesn't transfer across languages, and CodeBERT's very low Vul4J score (0.37 CodeBLEU, 0.12 CrystalBLEU)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about CodeBERT handling fragmented context better, CodeT5 excelling on complex patterns, and generalization struggles are all supported by Table 2 and the heatmap results in Sections 5.1-5.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'CodeT5's broad pre-training excels' and 'fine-tuning improves in-distribution performance' without controlling for confounds such as model size, architecture differences, or training data composition. No controlled experiments isolate these factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims to 'provide actionable insights to advance automated vulnerability patching for real-world security applications' but evaluation uses only BLEU-based similarity metrics, not functional correctness or security evaluation. The title and conclusions generalize beyond the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 considers that dataset characteristics (fragmented code, incomplete functions) explain performance differences. Section 7 discusses model randomness, hyperparameter sensitivity, and metric limitations as alternative factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Construct Validity) explicitly acknowledges: 'We evaluate correct patches using CodeBLEU and CrystalBLEU, which primarily gauge syntactic and limited semantic cues... they may overlook deeper security implications and potential exploit vectors.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers only to 'CodeBERT' and 'CodeT5' without specifying exact model checkpoints, sizes, or HuggingFace model IDs (e.g., microsoft/codebert-base, Salesforce/codet5-base)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses fine-tuning (not prompting). CodeBERT and CodeT5 are fine-tuned on vulnerability datasets in an encoder-decoder setup, not prompted."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported. Section 3.3 mentions fine-tuning but does not state learning rate, batch size, number of epochs, optimizer, or any training configuration."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are directly fine-tuned and evaluated in a standard sequence-to-sequence setup."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents three preprocessing steps (token length filtering at 512 tokens, comment removal via language-specific regex, normalization). Table 1 shows the number of rows affected at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' is a dedicated section with three subsections: Construct Validity, Internal Validity, and External Validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: CodeBLEU/CrystalBLEU may overlook security implications, token truncation may remove vital context, model randomness affects comparative outcomes, OOD performance drop reflects limited generalizability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 7 acknowledges limitations, it does not explicitly state what the results do NOT show. Statements like 'real-world projects frequently rely on specialized libraries and domain-specific coding styles' are generic rather than listing specific untested scenarios or claims the authors are not making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets are publicly available (Go/PHP on Zenodo, MegaVul on GitHub, Vul4J on GitHub, CodeParrot on HuggingFace). The authors' processed data is released via Zenodo [24]."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the six datasets used, their sources, and programming languages covered. Section 3.1 describes preprocessing steps."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data sources are standard public vulnerability datasets and code corpora."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 documents the pipeline: initial rows (Irows) → rows removed by tokenization (Rtok.) → rows removed by comment removal (Rcomm.) → rows removed by normalization (Rnorm.) → final rows (Trows) for each dataset."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses funding from the European Commission Horizon Europe Programme, LAZARUS project (Grant Agreement No. 101070303)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Luxembourg Institute of Science and Technology (LIST). They do not evaluate their own product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "EU Horizon Europe funding has no financial stake in whether CodeBERT or CodeT5 performs better on vulnerability repair tasks."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of when CodeBERT's or CodeT5's pre-training data was collected. The models were pre-trained on large code corpora but the training cutoff dates are not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.2 addresses overlap in their own train/test split but does not discuss whether CodeBERT's or CodeT5's pre-training data overlaps with the vulnerability datasets used for evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several datasets (MegaVul, Vul4J, CodeParrot) were publicly available before or around the time CodeBERT and CodeT5 were trained. No discussion of whether these models may have seen the evaluation data during pre-training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of automated models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates models on publicly available code datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example time is reported despite fine-tuning and evaluating models on an HPC cluster."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (Xeon Silver 4210, V100 GPUs) but total GPU hours, training time, or compute budget are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 acknowledges 'non-deterministic behavior (e.g., small variations in accuracy)' but no results across multiple seeds are reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. It is unclear whether any tuning was performed or what configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how final model configurations were selected. No validation set selection procedure is described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no correction for multiple comparisons is applied despite comparing across 6 datasets × 2 models × 2 metrics."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors create their own fine-tuning pipeline and preprocessing but do not acknowledge potential bias in their implementation choices or evaluation setup."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of compute budgets for either model. CodeBERT and CodeT5 differ in architecture and size but performance is not normalized by compute cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Construct Validity) explicitly discusses that CodeBLEU and CrystalBLEU 'primarily gauge syntactic and limited semantic cues' and 'may overlook deeper security implications and potential exploit vectors.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are fine-tuned and evaluated directly in a standard sequence-to-sequence setup."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CodeBERT/CodeT5's pre-training data includes code from after the vulnerability datasets were created, or vice versa."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the input format leaks information about the expected patch, or whether vulnerability labels provide unintended signal."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states 'all overlapping or duplicate instances were excluded' between train and test splits, partially addressing non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection method is applied (no canary strings, membership inference, n-gram overlap analysis, or temporal splits)."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeT5 consistently outperforms CodeBERT on datasets with complex vulnerability patterns (Vul4J, CodeParrot, MegaVul).",
       "evidence": "Table 2: Vul4J CodeBLEU CodeT5=0.9373 vs CodeBERT=0.3737; CodeParrot CodeBLEU CodeT5=0.9973 vs CodeBERT=0.997; MegaVul_C CodeBLEU CodeT5=0.8549 vs CodeBERT=0.8396 (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "CodeBERT performs better than CodeT5 on datasets with fragmented or incomplete code (Go, PHP).",
       "evidence": "Table 2: Go CodeBLEU CodeBERT=0.7641 vs CodeT5=0.6499; PHP CodeBLEU CodeBERT=0.7351 vs CodeT5=0.6924. Authors attribute this to CodeBERT's robustness with incomplete context (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning improves in-distribution performance but models fail to generalize to out-of-distribution datasets.",
       "evidence": "Figures 1 and 2 show heatmaps where diagonal (in-distribution) scores are high but off-diagonal (OOD) scores drop significantly. Section 5.2 discusses this pattern.",
       "supported": "moderate"
     },
     {
       "claim": "Models trained on one programming language transfer better to datasets in the same language than across languages.",
       "evidence": "Section 5.2: 'when a model trained on a specific programming language is tested on the same language—for example, trained on Megavul_C_2023 and tested on Megavul_C_2024—the accuracy remains high.' Similar for Vul4J/CodeParrot (both Java).",
       "supported": "moderate"
     },
     {
       "claim": "CodeT5 demonstrates superior scalability compared to CodeBERT.",
       "evidence": "Abstract and Section 5.1 state this but no scalability experiments (varying dataset size, model size, or compute) are presented to support it.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Duplicate datasets inflating diversity",
       "detail": "MegaVul_C_2023 and MegaVul_C_2024 have identical row counts (17,975 initial, 14,526 final), identical preprocessing stats, and nearly identical results (CodeBLEU: 0.8396/0.8395 CodeBERT, 0.8549/0.8549 CodeT5; CrystalBLEU: 0.7893/0.7893, 0.8131/0.8131). These appear to be the same dataset counted twice, inflating the claimed '6 datasets' to effectively 5."
     },
     {
       "flag": "No error bars despite acknowledged non-determinism",
       "detail": "Section 5.2 acknowledges 'non-deterministic behavior' and 'small variations in accuracy' but all results are single-point estimates. Without variance measures, it is impossible to determine whether observed differences are statistically meaningful."
     },
     {
       "flag": "Outdated models with no contemporary comparisons",
       "detail": "CodeBERT (2020) and CodeT5 (2021) are 4-5 years old. The paper does not compare against newer code LLMs (CodeT5+, StarCoder, DeepSeek-Coder, CodeLlama) or specialized vulnerability repair tools, limiting the practical relevance of the findings."
     },
     {
       "flag": "No functional correctness evaluation",
       "detail": "Evaluation uses only text similarity metrics (CodeBLEU, CrystalBLEU). Generated patches are never checked for compilation, test passage, or actual vulnerability remediation. A patch that is textually similar to the ground truth may not compile or may still be vulnerable."
     },
     {
       "flag": "Missing hyperparameters prevent reproduction",
       "detail": "No learning rate, batch size, number of epochs, optimizer, warmup schedule, or other training hyperparameters are reported, making the experiments unreproducible even with the released code."
     },
     {
       "flag": "Scalability claim without evidence",
       "detail": "The abstract claims 'CodeT5 also demonstrates superior scalability' but no scalability experiments are presented—no varying of dataset size, model size, or compute budget."
     }
   ],
   "cited_papers": [
     {
       "title": "APR4Vul: An Empirical Study of Automatic Program Repair Techniques on Real-World Java Vulnerabilities",
       "authors": ["Quang-Cuong Bui", "Ranindya Paramitha", "Duc-Ly Vu", "Fabio Massacci", "Riccardo Scandariato"],
       "year": 2024,
       "doi": "10.1007/s10664-023-10415-7",
       "relevance": "Empirical evaluation of APR techniques on real-world Java vulnerabilities, directly relevant to the survey's focus on automated vulnerability repair."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Foundational pre-trained code model evaluated in this study and widely used in LLM-based code understanding research."
     },
     {
       "title": "CodeT5: Identifier-Aware Unified Pre-Trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq Joty", "Steven C.H. Hoi"],
       "year": 2021,
       "arxiv_id": "2109.00859",
       "relevance": "Key pre-trained code generation model evaluated in this study, representative of encoder-decoder approaches to code tasks."
     },
     {
       "title": "Enhanced Automated Code Vulnerability Repair Using Large Language Models",
       "authors": ["David de-Fitero-Dominguez", "Eva García-López", "Antonio García-Cabot", "José-Javier Martínez-Herráiz"],
       "year": 2024,
       "doi": "10.1016/j.engappai.2024.109291",
       "relevance": "LLM-based vulnerability repair work that motivates the need for generalizable patch generation across vulnerability types."
     },
     {
       "title": "BERTVRepair: On the Adoption of CodeBERT for Automated Vulnerability Code Repair",
       "authors": ["N.N.H. Dang", "T.Q. Thanh", "A. Nguyen-Duc"],
       "year": 2024,
       "doi": "10.1007/978-3-031-55642-5_8",
       "relevance": "Directly studies CodeBERT adoption for vulnerability repair, closely related to this paper's evaluation scope."
     },
     {
       "title": "How Effective Are Neural Networks for Fixing Security Vulnerabilities",
       "authors": ["Yizheng Wu", "Nan Jiang", "Hoan Vu Pham", "Thibaud Lutellier", "Jeff Davis", "Lin Tan", "Petr Babkin", "Sameena Shah"],
       "year": 2023,
       "doi": "10.1145/3597926.3598135",
       "relevance": "Evaluates neural network effectiveness for security vulnerability repair, directly relevant to LLM-based automated program repair."
     },
     {
       "title": "Uncovering the Limits of Machine Learning for Automatic Vulnerability Detection",
       "authors": ["Niklas Risse", "Marcel Böhme"],
       "year": 2024,
       "relevance": "Investigates fundamental limits of ML for vulnerability detection, relevant to understanding generalization challenges in AI-based security."
     },
     {
       "title": "T5APR: Empowering Automated Program Repair Across Languages Through Checkpoint Ensemble",
       "authors": ["Reza Gharibi", "Mohammad Hossein Sadreddini", "Seyed Mostafa Fakhrahmad"],
       "year": 2024,
       "doi": "10.1016/j.jss.2024.112083",
       "relevance": "T5-based multi-language program repair approach, directly relevant to cross-language vulnerability repair evaluation."
     },
     {
       "title": "Patching as Translation: The Data and the Metaphor",
       "authors": ["Yangruibo Ding", "Baishakhi Ray", "Premkumar Devanbu", "Vincent J. Hellendoorn"],
       "year": 2021,
       "doi": "10.1145/3324884.3416587",
       "relevance": "Frames program repair as machine translation, foundational work for the sequence-to-sequence vulnerability patching approach used in this study."
     },
     {
       "title": "SequenceR: Sequence-to-Sequence Learning for End-to-End Program Repair",
       "authors": ["Zimin Chen", "Steve Kommrusch", "Michele Tufano", "Louis-Noël Pouchet", "Denys Poshyvanyk", "Martin Monperrus"],
       "year": 2021,
       "doi": "10.1109/TSE.2019.2940179",
       "relevance": "Early neural machine translation approach to program repair, directly relevant to the NMT lineage of automated code fixing."
     },
     {
       "title": "MegaVul: A C/C++ Vulnerability Dataset with Comprehensive Code Representations",
       "authors": ["Chao Ni", "Liyu Shen", "Xiaodan Yang", "Yan Zhu", "Shaohua Wang"],
       "year": 2024,
       "doi": "10.1145/3643991.3644886",
       "relevance": "One of the primary evaluation datasets used in this study; large-scale vulnerability dataset relevant to benchmarking code repair tools."
     },
     {
       "title": "GraphCodeBERT: Pre-Training Code Representations with Data Flow",
       "authors": ["Daya Guo", "Shuo Ren", "Shuai Lu"],
       "year": 2021,
       "relevance": "Pre-trained code model that extends CodeBERT with data flow analysis, relevant to understanding the landscape of code understanding models."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Evaluates models for vulnerability patching but provides no ready-to-use tool or clear actionable workflow for practitioners."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms expected results: fine-tuning helps in-distribution, OOD generalization is hard, larger pre-training helps on complex tasks."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Security-relevant topic (vulnerability repair) but no novel attack, no demonstration of dangerous capability gaps."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no provocative claims, straightforward benchmark comparison."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Zenodo artifact released but it's a research reproduction package, not a pip-installable tool or live demo."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors from LIST (Luxembourg), not a well-known AI lab. Models evaluated (CodeBERT, CodeT5) are established but not headline-generating."
     }
   }
 }