ai-research-survey

scan.json (26437B)

---

 {
   "paper": {
     "title": "CoDet-M4: Detecting Machine-Generated Code in Multi-Lingual, Multi-Generator and Multi-Domain Settings",
     "authors": ["Daniil Orel", "Dilshod Azizov", "Preslav Nakov"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.13733"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a HuggingFace link for data and pre-trained models: https://huggingface.co/datasets/DaniilOr/CoDET-M4 (Appendix A) and mentions 'We release our data and code' with a link in Section 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on HuggingFace: https://huggingface.co/datasets/DaniilOr/oDET-M4 (footnote 2) and https://huggingface.co/datasets/DaniilOr/CoDET-M4 (Appendix A). The paper explicitly states 'We release our data and code.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment setup section is provided in the paper. The paper mentions using vLLM for serving but does not specify library versions or a reproducible environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or 'Reproducing Results' section is provided in the paper. While the experimental setup is described, there are no concrete instructions to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results. All tables (Tables 2-14) report only point estimates for precision, recall, F1, and accuracy with no uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite multiple comparative claims (e.g., 'UniXcoder is superior', 'CodeT5 achieves better performance'). All comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. While raw performance numbers are provided, there is no systematic reporting of effect magnitudes beyond raw metric values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for sample sizes used in the various experiments. For example, the unseen models experiment uses only 126 samples, the unseen domains experiment uses 5,451 samples, and the unseen languages experiment uses 6,388 samples, but none of these choices are justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. The paper does not mention running experiments multiple times with different seeds or reporting any spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes a zero-shot baseline (Fast-DetectGPT), traditional ML baselines (SVM, CatBoost), and multiple DNN-based models (CodeBERT, CodeT5, UniXcoder) as described in Section 4.1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include contemporary models: Fast-DetectGPT (2024), UniXcoder (2022), CodeT5+ (2023), CodeBERT (2020). The paper also tests GPT-4o as a detector. These represent a reasonable range of contemporary approaches for this task."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed to identify which components of the proposed framework contribute most to performance. The paper compares different models but does not ablate specific design choices within any model (e.g., feature selection in SVM/CatBoost, or fine-tuning strategies in DNNs)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports four evaluation metrics: Macro F1 score (F), precision (P), recall (R), and accuracy (A), as stated in Section 4.1 ('Evaluation Measures')."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here since the task is binary/multi-class classification with ground-truth labels. The evaluation is appropriately automated using standard classification metrics against known labels."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is split into train/validation/test in an 8:1:1 ratio (Section 3.4), and results are reported on the test set. The out-of-domain experiments also use separate held-out datasets (unseen models, domains, languages)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive per-category breakdowns: per-language results (Table 3), per-source/domain results (Table 4), per-generator results (Tables 6, 8, Figure 8), per-language results for unseen languages (Table 10), and per-domain results for unseen domains (Table 11)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases including: CodeWhisperer being hard to detect (Section 4.5.1), performance drops in unseen domains (Section 4.5.2), JavaScript being most challenging (Section 4.5.3), and hybrid authorship detection failing (Section 4.5.4, Table 13). An error analysis of the baseline is provided in Appendix I."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: GPT-4o performs worse than traditional ML models for detection (Table 5, Section 4.3), significant performance drops in unseen domains (Table 9), hybrid authorship detection fails (Table 13), and performance degrades with increasing human code proportion (Figure 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the framework 'effectively distinguishes human-written from LLM-generated program code, setting a new benchmark for the task.' The results in Tables 2-4 support the in-domain detection claim (98.65% F1 for UniXcoder). The out-of-domain results (Tables 8-13) add nuance showing where the framework struggles, which is honestly reported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about why certain models perform better (e.g., 'UniXcoder depends on these structural elements to effectively capture relationships' in Section 4.5.2; 'CodeT5 appears to rely on more general patterns, making it more adaptable'). These are speculative explanations without controlled experiments to establish causation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to bound its generalizations. The Limitations section explicitly acknowledges that results are constrained to three main programming languages, function- and class-level code, and a narrow range of prompts. The conclusion also specifically notes performance drops in unseen domains and hybrid scenarios."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not systematically discuss alternative explanations for its results. For example, the high detection accuracy could partly be due to formatting artifacts from the generation process (comments/docstrings removed, length filtering), but this is not explored. The SHAP analysis in Appendix G touches on what features matter but does not discuss alternative explanations for the observed performance patterns."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "For the code generators, model sizes are specified (CodeLlama 7B, Llama3.1 8B, CodeQwen 1.5 7B, Nxcode-orpo 7B) but not specific version hashes or snapshot dates. GPT-4o is used without a version/snapshot identifier. For detector models (CodeBERT, UniXcoder, CodeT5), no specific model checkpoint versions are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendices C, E.1, E.2, E.3, E.4, and E.5 for all experimental settings: GPT-4o detection prompts, LeetCode generation prompts, CodeForces prompts, GitHub prompts, MBPP prompts, and hybrid generation prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Training hyperparameters are reported in Section 4.1: 5 epochs, learning rate 3e-4, weight decay 1e-3, batch size 256, linear LR scheduler for DNNs. CatBoost: 2,000 trees, learning rate 0.1. Generation: temperature 0.4-1.0 randomly. SVM: RBF kernel, primal formulation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach uses standard model fine-tuning and inference pipelines."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 (Quality Assurance) documents preprocessing: filtering CodeForces/LeetCode solutions that passed all test cases, removing HTML tags, filtering irrelevant LLM responses, removing comments/docstrings using regex, length filtering (excluding below 5th and above 95th percentile in token count), and deduplication."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after Section 5, covering four specific limitations: Generalizability (language constraints), Corpus Update (dataset obsolescence), Prompt Diversity, and Applied Models."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'Our research predominantly focuses on three programming languages' (generalizability), 'our study utilizes a narrow range of prompts' (prompt diversity), 'any dataset created to detect LLM-generated code can quickly become outdated' (corpus update), and 'We primarily relied on pre-existing models' (applied models)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: focuses on three main programming languages (Java, C++, Python), function- and class-level code, specific set of LLMs, and acknowledges the limitations of narrow prompt diversity. The Limitations section explicitly states what the dataset does NOT cover."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset is available on HuggingFace (https://huggingface.co/datasets/DaniilOr/CoDET-M4), allowing independent verification of the data used in experiments."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 (Data Collection) describes in detail: sources used (LeetCode, CodeForces via Kaggle, GitHub via CodeSearchNet and API), languages covered, how code was collected from each source, and the resulting sample sizes. Appendix A provides additional detail on the 3-month collection period (September-November 2024)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data consists of publicly available code samples and LLM-generated code. This is a standard benchmark study without human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: collection from multiple sources (Section 3.1), code generation process (Section 3.2), quality assurance with filtering steps (Section 3.3), and dataset splitting (Section 3.4). Table 1 provides detailed counts for each split by language and source."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper. The authors are from Mohamed bin Zayed University of Artificial Intelligence (MBZUAI), but no grants or funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed: all three authors are from Mohamed bin Zayed University of Artificial Intelligence (MBZUAI), UAE. No products of the authors' institution are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The absence of funding disclosure means this criterion cannot be satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the models used (CodeBERT, UniXcoder, CodeT5, or the code generators). This is relevant because the detector models are pre-trained on code corpora that may overlap with the test data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions deduplication within the dataset (Section 3.3) and ensures no overlap with The Vault repositories (Section 4.5.2), but does not discuss whether the pre-trained detector models (CodeBERT, UniXcoder, CodeT5) may have seen the evaluation code during their pre-training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The detector models (CodeBERT, UniXcoder, CodeT5) were pre-trained on code data that could include LeetCode solutions, CodeForces solutions, or GitHub code used in the evaluation. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported. The paper uses multiple LLMs for code generation (including GPT-4o API calls) and fine-tunes multiple models but does not report any cost information."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper mentions using vLLM for serving but does not report GPU hours, hardware used, training time, or total API spend."
       }
     }
   },
   "claims": [
     {
       "claim": "UniXcoder achieves the best binary classification performance with 98.65% F1 score for detecting LLM-generated code.",
       "evidence": "Table 2 shows UniXcoder achieving P=98.65, R=98.66, F=98.65, A=98.65, outperforming all other models (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o is ineffective at identifying LLM-generated code, even with few-shot learning, performing worse than traditional ML models.",
       "evidence": "Table 5 shows GPT-4o achieving at most 42.13% accuracy (3-shot), compared to SVM at 72.19% and CatBoost at 88.79% (Section 4.3, Table 2).",
       "supported": "moderate"
     },
     {
       "claim": "Models generalize well to unseen code generators, with UniXcoder achieving 93.22% F1 and 87.30% accuracy.",
       "evidence": "Table 8 shows UniXcoder achieving R=87.30, F=93.22, A=87.30 on code from 7 LLMs not in the training set (Section 4.5.1).",
       "supported": "weak"
     },
     {
       "claim": "Detection models experience significant performance drops in unseen domains, with the best model (CodeT5) achieving only 58.22% F1.",
       "evidence": "Table 9 shows CodeT5 at P=78.43, R=59.18, F=58.22, A=74.11 for unseen domains (Section 4.5.2).",
       "supported": "moderate"
     },
     {
       "claim": "UniXcoder generalizes relatively well to unseen programming languages, achieving 88.96% F1.",
       "evidence": "Table 12 shows UniXcoder at P=89.13, R=89.20, F=88.96, A=88.96 across four unseen languages (Section 4.5.3).",
       "supported": "moderate"
     },
     {
       "claim": "Detection models fail on hybrid authorship scenarios where LLMs complement or rewrite human-written code.",
       "evidence": "Table 13 shows UniXcoder achieving only R=33.22, F=39.36, A=64.71 on hybrid generated codes (Section 4.5.4).",
       "supported": "strong"
     },
     {
       "claim": "LLMs have unique writing styles that can be identified, enabling authorship attribution across generators.",
       "evidence": "Table 7 shows UniXcoder achieving 79.35% accuracy for 6-class authorship identification. Figure 2 shows confusion between related models (Nxcode/CodeQwen) as expected (Section 4.4).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CoDet-M4 introduces a 500K-sample multi-lingual, multi-generator, multi-domain dataset for detecting LLM-generated code. Fine-tuned pre-trained language models (UniXcoder, CodeT5) achieve near-perfect detection on in-domain data (98.65% F1), significantly outperforming traditional ML and zero-shot approaches including GPT-4o. However, detection performance drops substantially in out-of-domain settings: unseen code domains (55-58% F1), hybrid authorship scenarios (39% F1), and partially for unseen programming languages. The work highlights that current detection methods are brittle outside their training distribution, particularly for hybrid human-LLM code that reflects real-world usage patterns.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results across 14+ tables are reported as single-run point estimates with no confidence intervals, error bars, or standard deviations. The paper does not mention running experiments multiple times with different seeds, making it impossible to assess result stability."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Multiple comparative claims are made (e.g., UniXcoder vs. CodeT5, DNN vs. classical models) without any significance testing. Raw metric differences are treated as meaningful without assessing whether they could be due to chance."
     },
     {
       "flag": "Very small OOD test sets",
       "detail": "The unseen models experiment uses only 126 samples (Table 8). The unseen domains experiment uses 5,451 samples and unseen languages 6,388 samples, but no power analysis justifies these sizes for the claims being made."
     },
     {
       "flag": "Potential data artifacts from preprocessing",
       "detail": "Comments and docstrings were removed from all code samples, and length filtering excluded below 5th and above 95th percentile. These preprocessing choices could introduce systematic differences between human and LLM code that the models exploit, rather than detecting genuine stylistic differences. This confound is not discussed."
     },
     {
       "flag": "Pre-trained model contamination risk unaddressed",
       "detail": "CodeBERT, UniXcoder, and CodeT5 were pre-trained on large code corpora that likely include LeetCode and GitHub code used in the evaluation. The high in-domain performance could partly reflect data memorization rather than genuine detection capability."
     }
   ],
   "cited_papers": [
     {
       "title": "Distinguishing LLM-Generated from Human-Written Code by Contrastive Learning",
       "authors": ["Xiaodan Xu", "Chao Ni", "Xinrong Guo", "Shaoxuan Liu", "Xiaoya Wang", "Kui Liu", "Xiaohu Yang"],
       "year": 2025,
       "relevance": "Directly relevant prior work on detecting LLM-generated code using contrastive learning with UniXcoder-based semantic encoder."
     },
     {
       "title": "GPTSniffer: A CodeBERT-based classifier to detect source code written by chatgpt",
       "authors": ["Phuong T. Nguyen", "Juri Di Rocco", "Claudio Di Sipio", "Riccardo Rubei", "Davide Di Ruscio", "Massimiliano Di Penta"],
       "year": 2024,
       "relevance": "Prior work on binary classification to detect ChatGPT-generated code using CodeBERT, relevant baseline approach for machine-generated code detection."
     },
     {
       "title": "Automatic Detection of LLM-generated Code: A Case Study of Claude 3 Haiku",
       "authors": ["Musfiqur Rahman", "Sayed Hossein Khatoonabadi", "Ahmad Abdellatif", "Emad Shihab"],
       "year": 2024,
       "arxiv_id": "2409.01382",
       "relevance": "Case study on detecting Claude 3-generated code using ML models on CodeSearchNet dataset."
     },
     {
       "title": "Assessing AI Detectors in Identifying AI-Generated Code: Implications for Education",
       "authors": ["Wei Hung Pan", "Ming Jie Chok", "Jonathan Leong Shan Wong"],
       "year": 2024,
       "relevance": "Evaluation of AI detectors for code, revealing limitations in detection of LLM-generated code, used as foundation dataset in this work."
     },
     {
       "title": "Whodunit: Classifying code as human authored or GPT-4 generated – A case study on CodeChef problems",
       "authors": ["Oseremen Joy Idialu", "Noble Saji Mathews", "Rungroj Maipradit", "Joanne M. Atlee", "Mei Nagappan"],
       "year": 2024,
       "relevance": "Used stylometric features to identify GPT-4-generated code at class level, methodology adopted for SVM/CatBoost features in this work."
     },
     {
       "title": "M4: Multi-generator, Multi-domain, and Multi-lingual Black-Box Machine-Generated Text Detection",
       "authors": ["Yuxia Wang", "Jonibek Mansurov", "Petar Ivanov"],
       "year": 2024,
       "relevance": "Large-scale dataset for machine-generated text detection across domains, languages, and generators; inspired the multi-dimensional evaluation approach."
     },
     {
       "title": "Fast-detectgpt: Efficient zero-shot detection of machine-generated text via conditional probability curvature",
       "authors": ["Guangsheng Bao", "Yanbin Zhao", "Zhiyang Teng", "Linyi Yang", "Yue Zhang"],
       "year": 2024,
       "relevance": "Zero-shot AI-generated content detector used as the baseline in this paper's experiments."
     },
     {
       "title": "LLM-DetectAIve: a tool for fine-grained machine-generated text detection",
       "authors": ["Mervat Abassy", "Kareem Elozeiri", "Alexander Aziz"],
       "year": 2024,
       "relevance": "Tool for fine-grained detection of machine-generated text, relevant to the broader AI-generated content detection landscape."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark for code generation evaluation; foundational work for code LLM assessment."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "relevance": "Pre-trained model for code understanding used as one of the main detection models in this work."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free evaluation benchmark for code LLMs, relevant to benchmark contamination concerns in code generation evaluation."
     }
   ]
 }