ai-research-survey

scan.json (27816B)

---

 {
   "paper": {
     "title": "CODEMORPH: Mitigating Data Leakage in Large Language Model Assessment",
     "authors": [
       "Hongzhou Rao",
       "Yanjie Zhao",
       "Wenjie Zhu",
       "Ling Xiao",
       "Meizhen Wang",
       "Haoyu Wang"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2506.17627"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our artifact is available at https://github.com/security-pride/CodeMorph' in the Artifact Availability section after the contributions list."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark was constructed from The Stack dataset, which is publicly available, but the specific 100 selected code samples and the 720 code completion tasks constructed by the authors are not described as being released. The GitHub link references artifacts but no explicit dataset release is mentioned."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two NVIDIA A100 GPUs with 80GB of memory each, utilizing a Linux environment' (Section IV-B) but provides no requirements.txt, Dockerfile, or detailed dependency/library versions needed to reproduce the pipeline."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is provided but the paper does not describe how to run the experiments, what commands to execute, or how to replicate the benchmark construction process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '24.67% average reduction', '45% lower accuracy'). Figure 7 shows a distribution interval plot of similarity scores (box plots), but no confidence intervals or error bars are provided for the accuracy results in Figures 6, 8, or Tables III-IV."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'PESO results in an average accuracy reduction of 15%' vs random perturbation) but no statistical significance tests are reported. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout: 'accuracy decreased by an average of 24.67%, with Python showing the most significant reduction at 45%' (abstract), 'average reduction in similarity score of 7.01%' (Section V-B), and Table III shows accuracy for original, random, and PESO conditions."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 20 code samples per language (100 total) and acknowledges 'Due to the high cost associated with LLM API usage in CODEMORPH, we limit the selection to 20 code samples per group' (Section IV-A). This is an explanation but not a justification that N=20 is sufficient for the claims made. No power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance measures, or spread across runs are reported. The similarity score box plots in Figure 7 show distribution ranges, but accuracy results are single-run point estimates with no spread measures. Table IV shows per-sample scores but no aggregate variance statistics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes baselines: original (unperturbed) code performance as the primary baseline, and random perturbation as a baseline for comparing PESO's optimization (Table III, Table IV). These are appropriate comparison points."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare CODEMORPH against any contemporary code perturbation tools or decontamination methods. Related work discusses EvoEval (Xia et al. 2024), LiveCodeBench (Jain et al. 2024), and Liu et al. (2024)'s HumanEval expansion, but none are used as experimental baselines. The only comparisons are against no-perturbation and random perturbation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 serves as an ablation study, comparing PESO-guided perturbation against random perturbation method selection, isolating the contribution of the PESO optimization component. Results are in Tables III and IV."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation metrics: code completion accuracy, surface-level similarity (Levenshtein), semantic similarity (JPlag), and overall similarity score (weighted combination). These are reported in Tables III and IV and Figures 6-7."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is included: 'we opted for a human assessment to determine if the completed code performs the same functionality as the original code' (Section IV-A3) and 'we manually audited all perturbed code for semantic consistency with the original' (Section IV-B, Human Verification)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is constructed from The Stack dataset (StarCoder's training data), which is deliberately chosen to guarantee data leakage. The evaluation tests whether perturbation can mitigate this known contamination. The experimental design with confirmed contaminated data serves the paper's purpose, and the 720 completion tasks are a separate test set from the training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per programming language (C/C++, Python, Java, Rust, Go) in Figures 6-7 and Tables III-IV. Per-task breakdowns (1-line, 3-line, 5-line completions) are also provided in Table III and Figure 6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where perturbation did not reduce accuracy (Section V-A, 'code perturbation does not always reduce code completion accuracy'), with a specific example in Figure 8 showing cases where perturbed code completion succeeded but original failed. Section V-A also discusses correctness failures (4 errors in Java, 0-3 in other languages)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that single perturbation achieved 94.3% accuracy (almost no effect) in the preliminary study (Section II), that perturbation sometimes makes completion easier (Section V-A), and that random perturbation can sometimes outperform PESO (Section V-B: 'the random algorithm is not always less effective than PESO')."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (24.67% average accuracy decrease, 45% maximum for Python, 7.01% lower similarity for PESO, 15% average accuracy drop for PESO vs random, 25% maximum) are all supported by the results in Section V, Tables III-IV, and Figures 6-7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about CODEMORPH 'reducing' accuracy through perturbation. The study design is adequate: it uses a controlled comparison of original vs. perturbed code on the same completion tasks, and the PESO ablation compares random vs. optimized method selection with the same code samples. The single-variable manipulation (perturbation applied or not) supports causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to StarCoder and The Stack dataset. The Limitations section acknowledges 'we did not find other publicly available training datasets paired with corresponding LLMs for code, limiting this study to an evaluation of StarCoder' (Section VI). The title is appropriately scoped."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: perturbation may sometimes introduce hints that make completion easier (Section V-A), and the observation that accuracy reduction could be partly due to increased task difficulty rather than purely decontamination (Section V-A discusses both factors). The Limitations section addresses benchmark and model constraints."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'StarCoder with 15.5B parameters' and mentions 'GPT-4o', 'GPT-3.5-turbo', 'DeepSeek-coder', and 'Claude 3.5-sonnet' but provides no specific API versions, snapshot dates, or model version identifiers. Section III-D says 'GPT-4o' while Section IV-B says 'GPT-4' (inconsistency). No version strings like 'gpt-4o-2024-05-13' are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the prompt structure in natural language (Section III-A2: role prompt, task prompt, code prompt, answer prompt) but does not provide the actual prompt text used. Only the structure and purpose of each component is described, not the specific wording."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B reports key hyperparameters: mu=0.5, nu=0.5 for similarity weighting, T=2 for Boltzmann selection temperature, iteration limit=15, ssthreshold=0.2 for early stopping, and max input token limit=5k for StarCoder."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the full agentic pipeline in detail: the iterative perturbation-verification loop (Section III-A), the PESO optimization algorithm with Boltzmann selection (Section III-B), the multi-LLM voting verification system (Section III-A2), and the termination conditions (Section III-C). Algorithm 1 provides pseudocode."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A1 documents the data filtering pipeline: rule-based filtering (minimum 40 lines, 100 characters, compilability check) followed by manual selection for cross-file dependencies, multiple method definitions, or complex control flows, resulting in 100 source files across five languages."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI is titled 'LIMITATION' and contains two substantive subsections: 'Benchmark and Model' discussing the limited benchmark size and single-model evaluation, and 'Code Task' discussing the restriction to code completion tasks only."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'we did not find other publicly available training datasets paired with corresponding LLMs for code, limiting this study to an evaluation of StarCoder' and 'The Stack is a training dataset, lacking unit test cases, docstrings, and related information, making the construction of other code tasks challenging' (Section VI)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states specific scope boundaries: only StarCoder was evaluated, only code completion tasks were tested, the benchmark is relatively small due to API costs (Section VI). Future work directions indicate what was NOT tested (other datasets, other LLMs, other task types like code classification)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw data (100 selected code samples, 720 completion task definitions, per-task results) is not made available for independent verification. While The Stack dataset is public, the specific samples selected and the benchmark construction details are not released as data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A1 describes data collection in detail: starting from The Stack dataset, applying rule-based filtering (line count, character count, compilability), then manual selection for code with cross-file dependencies and complex structures, resulting in 100 samples across 5 languages."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for a study. The human verification of perturbed code appears to be done by the authors themselves, not recruited participants. The data source is a standard public dataset (The Stack)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in Section IV-A: The Stack → rule-based filtering (line/character thresholds, compilability) → manual selection → 100 source files → code task construction (1/3/5 line completion) → 720 tasks total. Figure 5 shows the benchmark construction process."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper. There is no mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all authors are from Huazhong University of Science and Technology, Wuhan, China, specifically the School of Cyber Science and Engineering."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The paper does not state whether it was funded or unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper's entire premise is about contamination. StarCoder's training data (The Stack) is explicitly stated and publicly available. The paper deliberately uses known-contaminated data: 'The StarCoder LLM has publicly released its training dataset, The Stack dataset, which is well-documented and confirmed to be contaminated' (Section II)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about train/test overlap. The benchmark is intentionally constructed from StarCoder's training data (The Stack) to guarantee contamination, and the paper's method is designed to mitigate this overlap through code perturbation. Section I and II discuss this extensively."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the central topic of the paper. The authors deliberately use contaminated data to test whether their perturbation approach can mitigate contamination effects. The entire methodology and evaluation is designed around this problem."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted. The human verification of code correctness was done as part of the methodology, not as a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted. The paper involves automated code perturbation and evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research study. The human reviewers appear to be the authors themselves performing code verification."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D states 'the cost of a single perturbation is approximately $0.20 per instance.' This provides per-instance cost information for the perturbation pipeline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the per-instance cost ($0.20) is stated and the hardware is mentioned (two NVIDIA A100 GPUs), the total computational budget (total API spend, total GPU hours, total wall-clock time for all experiments) is not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "CODEMORPH reduces LLM accuracy on code completion tasks by an average of 24.67% across five programming languages, with Python showing the most significant reduction at 45%.",
       "evidence": "Figure 6 shows accuracy distributions across single-line, three-line, and five-line code completion tasks for five languages. The largest gap is in Python single-line completion (Section V-A).",
       "supported": "moderate"
     },
     {
       "claim": "PESO achieves an average reduction in similarity score of 7.01% compared to random perturbation, with a maximum reduction of 42.86%.",
       "evidence": "Table IV presents detailed similarity score comparisons (surface-level, semantic, and overall) for 20 Python and 20 Go code samples, comparing PESO vs. random perturbation (Section V-B).",
       "supported": "moderate"
     },
     {
       "claim": "PESO reduces code completion accuracy by an average of 15% more than random perturbation, with a maximum reduction of 25%.",
       "evidence": "Table III shows accuracy comparisons for Python and Go across three task types. PESO accuracy is consistently lower than random perturbation accuracy (Section V-B).",
       "supported": "moderate"
     },
     {
       "claim": "CODEMORPH generates perturbed code with high accuracy and low correction cost, with at most 4 incorrect samples per language.",
       "evidence": "Section V-A reports 'four errors in Java, and between 0 and 3 errors in the remaining languages' after manual review, with reapplication typically requiring no more than two attempts.",
       "supported": "moderate"
     },
     {
       "claim": "Code similarity can effectively indicate decontamination, and reducing similarity is a practical strategy for achieving it.",
       "evidence": "Section V-B argues this based on the correlation between lower similarity scores (PESO) and lower accuracy in completion tasks (Table III and IV). However, this is shown on only 2 languages (Python, Go) with 20 samples each.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CODEMORPH applies 26 semantic-preserving code transformations using LLMs to perturb benchmark code, reducing StarCoder's code completion accuracy by an average of 24.67% across five programming languages. The PESO optimization algorithm, based on Boltzmann selection from genetic algorithms, selects more effective perturbation method combinations, achieving 7.01% lower similarity scores and 15% greater accuracy reduction compared to random perturbation selection. The approach supports C/C++, Python, Java, Rust, and Go while preserving cross-file dependencies and code compilability.",
   "red_flags": [
     {
       "flag": "Very small sample size",
       "detail": "Only 20 code samples per language (100 total) are used, justified by API cost constraints rather than statistical adequacy. The PESO ablation uses only 20 samples each in Python and Go. With N=20, observed differences could easily be due to sample selection effects."
     },
     {
       "flag": "Single model evaluation",
       "detail": "Only StarCoder (15.5B) is evaluated. The generalizability of CODEMORPH's decontamination effectiveness to other Code LLMs (e.g., GPT-4, CodeLlama, DeepSeek-Coder) is unknown. The authors acknowledge this limitation."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (CODEMORPH vs. original, PESO vs. random) are based on raw percentage differences without any statistical testing. Given N=20 per group, these differences may not be statistically significant."
     },
     {
       "flag": "No comparison with existing decontamination methods",
       "detail": "The paper discusses EvoEval, LiveCodeBench, and other decontamination approaches in related work but does not compare CODEMORPH against any of them experimentally. The only baseline is random perturbation."
     },
     {
       "flag": "Inconsistent model naming",
       "detail": "Section III-D refers to the perturbation LLM as 'GPT-4o' while Section IV-B calls it 'GPT-4'. These are different models with different capabilities and costs. It is unclear which was actually used."
     },
     {
       "flag": "Human evaluation details sparse",
       "detail": "The paper mentions human assessment for code completion correctness and manual auditing of perturbed code, but does not describe who performed the evaluation, whether inter-rater agreement was measured, or what guidelines were followed."
     }
   ],
   "cited_papers": [
     {
       "title": "Quantifying contamination in evaluating code generation capabilities of language models",
       "authors": ["M. Riddell", "A. Ni", "A. Cohan"],
       "year": 2024,
       "arxiv_id": "2403.04811",
       "relevance": "Directly relevant to benchmark contamination in code LLM evaluation, which CODEMORPH builds upon for its similarity scoring methodology."
     },
     {
       "title": "Concerned with data contamination? assessing countermeasures in code language model",
       "authors": ["J. Cao", "W. Zhang", "S.-C. Cheung"],
       "year": 2024,
       "arxiv_id": "2403.16898",
       "relevance": "Studies data perturbation as a countermeasure against data contamination in code LLMs, foundational to CODEMORPH's approach."
     },
     {
       "title": "Rethinking benchmark and contamination for language models with rephrased samples",
       "authors": ["S. Yang", "W.-L. Chiang", "L. Zheng", "J. E. Gonzalez", "I. Stoica"],
       "year": 2023,
       "arxiv_id": "2311.04850",
       "relevance": "Demonstrates data leakage in HumanEval and MBPP, motivating the need for decontamination methods like CODEMORPH."
     },
     {
       "title": "Livecodebench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["N. Jain", "K. Han", "A. Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "A continuously updated benchmark designed to avoid contamination, representing an alternative approach to the perturbation-based decontamination studied here."
     },
     {
       "title": "Top leaderboard ranking=top coding proficiency, always? evoeval: Evolving coding benchmarks via llm",
       "authors": ["C. S. Xia", "Y. Deng", "L. Zhang"],
       "year": 2024,
       "arxiv_id": "2403.19114",
       "relevance": "Uses LLMs to evolve coding benchmarks to mitigate contamination, a closely related approach to CODEMORPH's perturbation strategy."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, one of the most widely used code generation benchmarks, which has known contamination issues discussed in this paper."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2024,
       "relevance": "Expanded HumanEval test cases to increase evaluation rigor, a perturbation-based decontamination approach."
     },
     {
       "title": "Dynamic evaluation of large language models by meta probing agents",
       "authors": ["K. Zhu", "J. Wang", "Q. Zhao", "R. Xu", "X. Xie"],
       "year": 2024,
       "relevance": "Proposes dynamic evaluation using psychometrics-inspired reconstruction to assess LLMs, related to contamination mitigation."
     },
     {
       "title": "Benchmark data contamination of large language models: A survey",
       "authors": ["C. Xu", "S. Guan", "D. Greene", "M. Kechadi"],
       "year": 2024,
       "arxiv_id": "2406.04244",
       "relevance": "Survey of benchmark data contamination in LLMs, providing context for the contamination problem CODEMORPH addresses."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["T. Y. Zhuo", "M. C. Vu", "J. Chim"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Repository-level code generation benchmark that faces similar contamination concerns addressed by CODEMORPH."
     },
     {
       "title": "Recode: Robustness evaluation of code generation models",
       "authors": ["S. Wang", "Z. Li", "H. Qian"],
       "year": 2022,
       "relevance": "Develops semantics-preserving code perturbation methods for evaluating code generation model robustness, directly related to CODEMORPH's transformation approach."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu"],
       "year": 2023,
       "relevance": "Comprehensive survey of LLMs in software engineering, providing broader context for code LLM evaluation methodology."
     }
   ]
 }