ai-research-survey

scan.json (31118B)

---

 {
   "paper": {
     "title": "On the Reliability and Explainability of Language Models for Program Generation",
     "authors": [
       "Yue Liu",
       "Chakkrit Tantithamthavorn",
       "Yonghui Liu",
       "Li Li"
     ],
     "year": 2023,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2302.09587",
     "doi": "10.1145/3641540"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Evaluating 8 pre-trained language models on 5 program generation benchmarks reveals that performance is substantially inflated by data duplication between training and testing sets (>20% identical test samples in several datasets). Models frequently copy input sequences rather than generating new code — up to 80% output-input duplication for some models on code review tasks. SHAP-based explainability analysis shows models prioritize identifiers and keywords but exhibit poor robustness even to removal of supposedly unimportant tokens.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we publish the studied dataset and a replication package, which are publicly available on GitHub' with a concrete URL: https://github.com/yueyueL/ProgramGen-LMs-Reliability (footnote 1, Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All five benchmark datasets used are publicly available (CodeXGLUE, Tufano et al., CodeReview), and the replication package includes the studied datasets. Section 3.2 describes each dataset with public sources."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.4 lists hardware (AMD Ryzen 9 5950X, 64GB RAM, NVIDIA RTX 3090) and mentions PyTorch, Transformers, Captum, and Ecco libraries, but no library version numbers, requirements.txt, or Dockerfile are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub replication package (footnote 1) and Section 3.4 describes the experimental setup including model configurations, data splits from CodeXGLUE, and fine-tuning procedures. The replication package is explicitly intended for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 and all performance tables report only point estimates (e.g., '15.20% accuracy') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 8 models across 12 datasets and makes claims like 'CodeT5+ shows superior performance' based solely on comparing raw accuracy numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance differences are consistently reported with baseline context, e.g., Table 3 shows 'Original Accuracy: 70.60% → New Accuracy: 53.09%' when removing high-similarity instances, and 'accuracy of CodeT5+ decreases from 15.2% to 9.19%' for token removal (Section 4.3.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses standard benchmark datasets without justifying why these sample sizes are appropriate or adequate for the claims made. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single experimental runs. No standard deviations, variance across seeds, or spread measures are reported anywhere in the paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Eight models are compared, including T5 (NL-only pre-training) as a baseline against code-specialized models. Section 3.1 describes the rationale for including encoder-only, decoder-only, and encoder-decoder architectures for completeness."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include CodeT5+ (2023), CodeReviewer (2022), CodeT5 (2021), and other recent models. These represent the state of the art at the time of writing."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not propose a multi-component system — it is an empirical evaluation of existing pre-trained models. The token removal experiments (Section 4.3.2) are robustness analyses, not component ablations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses both accuracy (exact match rate) and BLEU-4 score. Section 3.3 describes both metrics and Table 3 reports both for CodeReviewer and CodeT5+."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality is performed. All evaluation is automated (accuracy, BLEU, SHAP analysis). Human evaluation of code correctness and usefulness would be relevant to reliability claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states 'For each dataset, we use identical training, validation, and test splits as well as fine-tuning approaches described in CodeXGLUE and original papers.' Separate test sets are used for evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-dataset breakdowns across all 12 subsets. Figures 2 and 3 further break down performance by similarity score ranges. Figure 5 breaks down feature importance by token type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 discusses the failure mode of models copying inputs. Figure 4 shows concrete examples of test instances with duplicated sources and different targets. Section 4.3 uses XAI to analyze failure patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core contribution is negative results: performance inflation from data duplication, models merely copying inputs (Table 4), significant performance drops from minor input changes (Figure 6), and overall low accuracy on most tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (1) 'inappropriate performance evaluation stemming from severe data duplication, causing over-optimistic results' — supported by RQ2 (Section 4.2, Tables 3-4), and (2) 'models can recognize code grammar and structural information, but they exhibit limited robustness' — supported by RQ3 (Section 4.3, Figures 5-7)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about data duplication inflating performance, supported by a controlled experiment: removing high-similarity instances and measuring performance drop (Table 3). The token removal study (Section 4.3.2) uses controlled single-variable manipulation to establish causal relationships between tokens and output."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'On the Reliability and Explainability of Language Models for Program Generation' and conclusions like 'Automated program generation models are not reliable' generalize beyond the 8 tested models and 5 datasets. The study primarily covers Java/C# with specific model architectures, but conclusions are framed broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 discusses threats related to model settings and dataset selection but does not discuss alternative explanations for the findings. For example, it does not consider whether performance drops from removing duplicates might reflect difficulty differences in the remaining test set rather than memorization."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central thesis IS about the gap between benchmark metrics (accuracy/BLEU) and actual model reliability. The paper explicitly argues that high benchmark accuracy is a poor proxy for practical reliability, providing extensive evidence (RQ2) that measured performance does not reflect real capability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by name (CodeT5, CodeBERT, etc.) with '12-layer base version' specified (Section 3.4), but exact HuggingFace model paths or checkpoint IDs are not provided. For CodeTrans, the paper says 'the version with the most downloads,' which is not reproducibly specific."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. All models are fine-tuned encoder-decoder or encoder/decoder models that directly generate code from input sequences."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.4 mentions beam search size of 1 and references CodeXGLUE and original papers for fine-tuning settings, but key hyperparameters (learning rate, batch size, epochs, optimizer) are not stated in this paper."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are fine-tuned and run standard encoder-decoder inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes each dataset's structure, splits, and sizes in detail. Section 3.4 describes the identical data splits from CodeXGLUE and original papers. The analysis methodology (BLEU-4 similarity computation, token type classification, special token handling) is described in Sections 4.2-4.3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 'Threats to Validity' provides a substantive discussion of internal and external threats, covering model architecture choices, hyper-parameter settings, dataset selection, and generalizability."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses specific threats: 'The primary threat to internal validity mainly lies in the model architecture and hyper-parameter setting. We use eight program generation models, which are based on the same model settings in the original papers.' It also notes specific dataset coverage (four task types, multiple programming languages, three input types)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it did NOT test. It does not mention excluding large language models (GPT-3.5/4, ChatGPT), larger model sizes, or additional programming languages. Section 5.2 implies scope limitations but does not state specific exclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper publishes a replication package on GitHub (footnote 1) including the studied datasets, and all five benchmark datasets are publicly available from their original sources."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes each dataset's collection process by referencing original papers: Tufano et al. collected from Gerrit repositories, Bugs2Fix from GitHub bug-fixing commits, CodeTrans from CodeXGLUE, CONCODE from ~33,000 Java projects. Collection details are traced to original sources."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from datasets to analysis is documented: standard splits are used (Section 3.4), BLEU-4 similarity computation is formalized with equations (Equations 1-4 in Section 4.2), and token classification for explainability is described in Section 4.3.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section states: 'Chakkrit Tantithamthavorn was supported by the Australian Research Council's Discovery Early Career Researcher Award (DECRA) funding scheme (DE200100941).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Monash University (Australia) and Beihang University (China). None of the authors are affiliated with companies whose models are evaluated (all are open-source academic models)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Australian Research Council is a government funding body with no financial interest in the outcome of the model evaluations."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper extensively discusses fine-tuning train/test overlap, it does not state the pre-training data cutoff dates for the base models (CodeT5, CodeBERT, etc.), which could have been pre-trained on data containing the benchmark examples."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's primary contribution. RQ2 (Section 4.2) provides a comprehensive analysis of train-test overlap using BLEU-4 similarity scoring, finding >20% of test samples are identical to training samples in several datasets (Tufano et al., CodeTrans-Dataset)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire RQ2 addresses benchmark contamination. Section 4.2.1 analyzes train-test duplication with formal metrics (Equations 1-2), Section 4.2.2 examines intra-test-set duplication, and Section 4.2.3 analyzes output-input similarity. Table 3 quantifies performance inflation from contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a computational evaluation of language models on benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example time is reported despite fine-tuning and running inference on 8 models across 12 dataset configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.4 lists hardware (RTX 3090, AMD Ryzen 9, 64GB RAM) but does not state total GPU hours, training time, or overall computational budget for the 96 model-dataset experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single experimental runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many runs produced the results. No mention of 'averaged over K runs' or similar."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper states it uses original paper settings and 'the goal of our work is not to find the best setting' (Section 5.2) but does not report any search budget or confirm that no tuning was performed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states 'we use identical training, validation, and test splits as well as fine-tuning approaches described in CodeXGLUE and original papers.' Using standard configurations from prior work is a justified selection strategy for a comparative study."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 8 models across 12 datasets, making numerous comparisons, but performs no statistical tests at all — and therefore no multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement fine-tuning and evaluation pipelines for all 8 models but do not discuss potential bias from their own implementation choices affecting baseline performance."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While the paper controls for model size by using 12-layer base versions (Section 3.4), performance is not reported as a function of compute budget, and compute differences between models are not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's central contribution. RQ2 directly questions whether benchmark metrics reflect real model capability, finding that data duplication, output-input copying, and test set redundancy undermine construct validity of program generation benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are fine-tuned and run standard sequence-to-sequence inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss temporal leakage — whether pre-trained models' training data temporally overlaps with the benchmark datasets. The analysis focuses on fine-tuning train/test overlap, not pre-training data contamination."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Feature leakage is not specifically discussed. The paper does not analyze whether evaluation inputs leak answer information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 extensively addresses non-independence: Section 4.2.1 quantifies train-test overlap using BLEU-4 similarity (Equations 1-2), and Section 4.2.2 quantifies intra-test-set duplication (Equations 3-4). Finding 2 and Finding 3 detail the prevalence across all 12 datasets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses BLEU-4 scoring as a concrete leakage detection method (Equations 1-4), computing maximum pairwise similarity scores between test and training instances to identify exact and near-duplicate contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "Pre-trained encoder-decoder models (CodeReviewer, CodeT5+) show superior performance compared to encoder-only and decoder-only models across diverse program generation datasets.",
       "evidence": "Table 2 shows CodeT5+ and CodeReviewer consistently achieving highest accuracy across most of the 12 datasets, e.g., CodeReviewer at 30.43% on CodeReview and CodeT5+ at 70.6% on C#2Java (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Multiple program generation benchmark datasets contain substantial data duplication between training and testing sets, inflating performance metrics.",
       "evidence": "Figure 2 and Table 3 show >20% of test samples are identical to training data in Tufano et al. datasets. Removing high-similarity instances (>0.6 BLEU) causes CodeT5+ accuracy to drop from 70.6% to 53.09% on C#2Java (Section 4.2.1).",
       "supported": "strong"
     },
     {
       "claim": "10 of 12 studied datasets contain duplicated source sequences within their test sets, requiring models to generate different targets for identical inputs.",
       "evidence": "Figure 3 shows intra-test-set duplication across datasets, with Android_S having 11% duplicates and Ovirt_M over 10%. Figure 4 provides a concrete example from Android_S (Section 4.2.2).",
       "supported": "strong"
     },
     {
       "claim": "Language models frequently generate outputs identical to input sequences in code review and repair tasks, rather than generating updated code.",
       "evidence": "Table 4 shows output-input duplication rates up to 84% (CodeTrans on B2F_M), 80% (T5 on Android_M), and 35% (CodeT5+ on B2F_S). Only CONCODE (text-to-code) shows 0% duplication (Section 4.2.3).",
       "supported": "strong"
     },
     {
       "claim": "Identifiers and keywords are consistently assigned higher feature importance than operators and separators by program generation models.",
       "evidence": "Figure 5 shows average feature importance of token types across CodeT5, CodeReviewer, and CodeT5+ on 10 datasets, with identifiers and keywords consistently receiving higher SHAP importance scores (Section 4.3.1).",
       "supported": "moderate"
     },
     {
       "claim": "Language models exhibit poor robustness to input changes — removing even a single least-important token significantly degrades performance.",
       "evidence": "Figure 6 shows CodeT5+ accuracy drops from 15.2% to 9.19% on Android_S after removing just one token, and to 0% after removing 10 tokens. Figure 7 confirms the pattern across all 12 datasets for three models (Section 4.3.2).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "The paper compares 8 models across 12 datasets and draws conclusions about relative performance without any statistical significance tests. All comparisons are based on raw accuracy numbers, making it impossible to determine whether observed differences are meaningful or due to random variation."
     },
     {
       "flag": "No variance or multi-run reporting",
       "detail": "All results appear to be from single experimental runs with no standard deviations, confidence intervals, or cross-seed variation reported. For fine-tuned neural models, single-run results can be unreliable due to random initialization and training stochasticity."
     },
     {
       "flag": "Over-broad conclusions from limited scope",
       "detail": "The paper concludes 'Automated program generation models are not reliable' (Section 7) based on 8 specific models (all relatively small, 12-layer base versions) on 5 datasets. Large language models like GPT-3.5/4 and larger model variants are not studied, yet conclusions are stated without qualification."
     }
   ],
   "cited_papers": [
     {
       "title": "The adverse effects of code duplication in machine learning models of code",
       "authors": ["Miltiadis Allamanis"],
       "year": 2019,
       "relevance": "Directly relevant finding that code duplication inflates ML model performance metrics by up to 100%, foundational work for this paper's RQ2."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq Joty", "Steven CH Hoi"],
       "year": 2021,
       "arxiv_id": "2109.00859",
       "relevance": "State-of-the-art pre-trained code model evaluated in the study, widely used in code generation research."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "relevance": "Foundational encoder-based pre-trained code model evaluated in the study."
     },
     {
       "title": "CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation",
       "authors": ["Shuai Lu", "Daya Guo", "Shuo Ren"],
       "year": 2021,
       "relevance": "Benchmark suite used for datasets and evaluation protocol in this study; its data quality is a subject of the findings."
     },
     {
       "title": "Natural attack for pre-trained models of code",
       "authors": ["Zhou Yang", "Jieke Shi", "Junda He", "David Lo"],
       "year": 2022,
       "relevance": "Demonstrates adversarial vulnerability of pre-trained code models, complementary to this paper's robustness analysis."
     },
     {
       "title": "An extensive study on pre-trained models for program understanding and generation",
       "authors": ["Zhengran Zeng", "Hanzhuo Tan", "Haotian Zhang"],
       "year": 2022,
       "relevance": "Empirical evaluation of pre-trained code models suggesting more rigorous evaluations are needed, closely related work."
     },
     {
       "title": "Pitfalls in Language Models for Code Intelligence: A Taxonomy and Survey",
       "authors": ["Xinyu She", "Yue Liu", "Yanjie Zhao"],
       "year": 2023,
       "arxiv_id": "2310.17903",
       "relevance": "Survey of common experimental biases in LM-for-code research, including data noise and labeling errors."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2023,
       "arxiv_id": "2308.10620",
       "relevance": "Systematic review of LLMs in software engineering, providing survey-level context for code generation capabilities."
     },
     {
       "title": "Codet5+: Open code large language models for code understanding and generation",
       "authors": ["Yue Wang", "Hung Le", "Akhilesh Deepak Gotmare"],
       "year": 2023,
       "arxiv_id": "2305.07922",
       "relevance": "Enhanced pre-trained code model evaluated as top performer in this study."
     },
     {
       "title": "On learning meaningful code changes via neural machine translation",
       "authors": ["Michele Tufano", "Jevgenija Pantiuchina", "Cody Watson", "Gabriele Bavota", "Denys Poshyvanyk"],
       "year": 2019,
       "relevance": "Source of the Tufano et al. code review dataset found to have severe train-test duplication in this study."
     },
     {
       "title": "You autocomplete me: Poisoning vulnerabilities in neural code completion",
       "authors": ["Roei Schuster", "Congzheng Song", "Eran Tromer", "Vitaly Shmatikov"],
       "year": 2021,
       "relevance": "Demonstrates poisoning attacks on code completion models, relevant to reliability and security of code generation."
     },
     {
       "title": "On the importance of building high-quality training datasets for neural code search",
       "authors": ["Zhensu Sun", "Li Li", "Yan Liu", "Xiaoning Du"],
       "year": 2022,
       "relevance": "Demonstrates that data quality significantly impacts code intelligence model performance, supporting this paper's data quality findings."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Findings about benchmark duplication and model unreliability are directly useful for researchers evaluating or building code generation models."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Reveals that widely-used benchmark results are inflated by severe data duplication, and models often just copy inputs — challenging the perceived effectiveness of code generation models."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No direct AI safety or security concerns raised; the paper is about evaluation methodology, not harmful capabilities."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Some 'benchmarks are broken' angle — finding that prior performance claims are inflated by dataset issues — but framed academically rather than confrontationally."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Replication package available on GitHub, but no interactive demo or easily runnable tool for practitioners."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Academic authors from Monash University and Beihang University; models studied (CodeT5, CodeBERT) are known in the research community but not household names."
     }
   }
 }