ai-research-survey

scan.json (28426B)

---

 {
   "paper": {
     "title": "LiCoEval: Evaluating LLMs on License Compliance in Code Generation",
     "authors": ["Weiwei Xu", "Kai Gao", "Hao He", "Minghui Zhou"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2408.02487",
     "doi": "10.48550/arXiv.2408.02487"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Even top-performing LLMs produce 0.88–2.01% of code strikingly similar to existing open-source implementations, and most fail to provide accurate license information, particularly for copyleft licenses (only Claude-3.5-Sonnet provides any copyleft license info). The paper establishes a four-criteria striking similarity standard validated by expert review (8 reviewers, 32/33 precision) and introduces LiCoEval, the first benchmark for evaluating LLM license compliance capability across 14 models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is released at https://github.com/osslab-pku/LiCoEval as stated in Section VII and reference [92]."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "LICOEVAL benchmark (4,187 function-level code snippets) is made publicly available at the same repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specifications are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the methodology but does not include scripts or commands to replicate the evaluation."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table IV are point estimates (counts, percentages, accuracy scores) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Mann-Whitney U test is used in Section III-E (Table II) to compare ACCESSED vs UNSEEN groups, with Cliff's Delta effect size test. However, the main 14-model evaluation in Section V reports raw comparisons without significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cliff's Delta effect sizes are reported in Table II with interpretation levels (all negligible). Section V reports percentages of strikingly similar cases providing context for magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 10,000 samples per group and 4,187 benchmark samples but provides no explicit justification for these sizes and no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results use greedy decoding (temperature=0) with single-run numbers. No variance across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "14 LLMs are compared against each other, and the ACCESSED vs UNSEEN comparison serves as the baseline for establishing the striking similarity standard."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation includes 2024-era models: GPT-4o, Claude-3.5-Sonnet, DeepSeek-Coder-V2, Gemini-1.5-Pro, and other recent models (Table IV)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The striking similarity standard has four criteria (body lines, complexity, text similarity, identical comments) but no formal ablation study systematically removes individual criteria to measure their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: BLEU-4, Jaccard similarity, edit distance for text similarity; cyclomatic complexity, body lines, comment similarity for characterizing striking similarity; and the composite LICO score for compliance."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Eight expert reviewers (5 developers with >6 years experience, 3 IP lawyers) evaluated 33 strikingly similar code pairs in Section III-F, identifying 32/33 as cases where independent creation can be excluded."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "ACCESSED_EVAL and UNSEEN_EVAL groups (10,000 samples each, non-overlapping with original groups) were constructed for validation, tested with WizardCoder and Poro-34B-chat (Section III-F)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by license type (permissive, weak copyleft, strong copyleft) in Figure 8 and Table IV, and per-model results are provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 shows Example-2, a case that falls below the striking similarity standard (ambiguous case). Models that completely fail at compliance are discussed (DeepSeek-Coder-V2 accuracy=0, WizardCoder-Python-13B LICO=0.153)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that text similarity alone is insufficient to determine non-independent creation (Section III-E). Most models fail completely on copyleft license compliance. The standard's potential weakness on recall is acknowledged."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 0.88–2.01% strikingly similar code are supported by Table IV. The claim that most LLMs fail on copyleft licenses is supported by Table IV showing Accc=0 for all models except Claude-3.5-Sonnet."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-adjacent claims, e.g., attributing StarCoder2's zero copyleft violations to its 'file-level, fine-grained license detection strategy' (Section VI-A1) based on observational evidence from a single model without controlled comparison."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Code Generation' generally and the abstract says 'code generation tasks,' but the study only covers Python function-level code. The threats section acknowledges 'our study primarily focused on Python code' and 'We only addressed function-level code completion,' but the title and abstract overstate the scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for its main findings. For example, it does not consider whether model architecture differences (not just training data) could explain compliance variation, or whether the striking similarity standard thresholds are sensitive to the specific choices made."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between 'striking similarity' (what they measure) and 'copying relationship' / 'non-independent creation' (what they claim), stating their standard is a 'preliminary standard' that is 'not intended to establish definitive legal boundaries' (Sections III-A, III-F)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed by marketing names only (GPT-3.5-Turbo, GPT-4-Turbo, GPT-4o, Claude-3.5-Sonnet, Gemini-1.5-Pro) without specific API versions or snapshot dates. Open-source models include sizes (e.g., Qwen2-7B-Instruct) but still lack version specifics."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt construction (file header + imports + signature + docstring) and mentions 'one-shot approach' but does not provide the actual one-shot example, system instructions, or the follow-up prompt used to ask for license information."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is set to 0 with greedy decoding, stated in Section III-C and Section V-A."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is direct prompting of LLMs for code completion."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Detailed preprocessing pipeline is documented: function extraction from Starcoderdata (74.7M → 2.6M after filtering → 10K sampled), UNSEEN construction with MinHash/LSH dedup (Jaccard threshold 0.2), LICOEVAL construction from WoC (700M blobs → filtering → 10K candidates → 4,187 final). Counts at each stage provided."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B 'Threats to validity' provides substantive discussion with internal validity (Section VI-B1) and external validity (Section VI-B2) subsections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: the standard 'focuses on precision, potentially overlooking cases' where LLMs generate derived code below threshold; LICOEVAL's 4,187 samples 'may not fully represent the vast diversity of real-world code'; Python-only focus; function-level only limitation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'primarily focused on Python code' (VI-B2), 'only addressed function-level code completion' (VI-B2), 'findings are not intended to establish definitive legal boundaries' (III-A), and the standard 'may perform poorly on recall' (VI-B1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "LICOEVAL benchmark is publicly available at https://github.com/osslab-pku/LiCoEval (reference [92])."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed data collection from Starcoderdata, CodeParrot-clean, and World of Code (version U, Oct 2021) with explicit filtering criteria and processing steps described in Sections III-D1 and IV-B1."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The expert panel (5 developers, 3 lawyers) is characterized by qualifications ('over six years of coding experience,' 'specializing in software intellectual property') but the recruitment channel and method are not described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented with counts: 74.7M functions → 2.6M after filtering → 10K sampled for ACCESSED; separate pipeline for UNSEEN with LSH dedup yielding 157,273 candidates → 10K sampled; LICOEVAL construction from 700M WoC blobs → filtering → 10K candidates → 4,187 final."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgment section: 'This work is sponsored by the National Natural Science Foundation of China 62332001.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Peking University, University of Science and Technology Beijing, Carnegie Mellon University. No authors are affiliated with the LLM companies being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is NSFC (Chinese government science foundation), which has no financial stake in the evaluation outcomes of any specific LLM."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the 14 evaluated LLMs. The paper only discusses WizardCoder's training data (The Stack) in the empirical study."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire empirical study (Section III) is about distinguishing accessed vs unseen training data. The benchmark is deliberately constructed from widely-reused code likely in training data. For WizardCoder, overlap is explicitly characterized using MinHash/LSH."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's methodology explicitly addresses contamination by design: UNSEEN group uses code verified absent from WizardCoder's training set via MinHash/LSH (Jaccard threshold 0.2), and the striking similarity standard is calibrated to distinguish memorized from independently created code."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a benchmark evaluation study. The expert validation panel is a methodological tool for standard validation, not a pre-registerable human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The paper is primarily a benchmark evaluation. The expert panel serves as validators of the striking similarity standard, not as research subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a benchmark evaluation study. The expert panel's qualifications are described but this is not a human subjects study requiring demographic reporting."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. Expert validators were selected for domain expertise."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with human participants assigned to conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects experiment. The expert validation was conducted 'without knowledge of their origins' but this is standard validation methodology, not a blinding protocol for a human study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. All 8 expert validators completed their evaluations."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token usage, or wall-clock time are reported for running evaluations on 4,187 benchmark samples across 14 models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware constraints are mentioned as a factor in selecting WizardCoder but specific hardware, GPU hours, or total compute budget are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy decoding (temperature=0) is used, making results deterministic. No seed sensitivity analysis is performed or discussed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs is not explicitly stated. Greedy decoding implies a single deterministic run, but this is not stated explicitly."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The striking similarity thresholds were derived from empirical analysis but no systematic search over alternative thresholds was documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The striking similarity standard (4 criteria) is empirically derived from ACCESSED vs UNSEEN comparison and validated on held-out data (ACCESSED_EVAL, UNSEEN_EVAL) achieving 100% precision across 33 samples from 2 models."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Four Mann-Whitney U tests are run (Table II) with all p-values reported as <0.01, but no multiple comparison correction (Bonferroni, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed LICOEVAL and evaluate models on it without discussing the potential bias of evaluating on their own benchmark or whether the benchmark's design choices favor certain types of models."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 7B to very large (GPT-4) with vastly different compute requirements, but performance is not discussed as a function of compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper validates the benchmark through the empirical study (Section III) establishing the striking similarity standard, expert review (8 reviewers confirming 32/33 cases), and held-out validation (ACCESSED_EVAL/UNSEEN_EVAL with 100% precision). Limitations of precision vs recall are discussed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All models are evaluated with direct one-shot prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses code from WoC version U (October 2021) and widely-reused code that predates most models' training. However, temporal leakage implications for the 14 evaluated models are not explicitly discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The prompt includes file header comments (which may contain license information for LICOEVAL's code files). When the LLM is later asked to provide license info in the follow-up inquiry, it may have already seen the license in the prompt context. This potential feature leakage is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The empirical study explicitly addresses non-independence: UNSEEN group is verified absent from training data via MinHash/LSH dedup (Jaccard threshold 0.2, more stringent than the 0.5 used in Starcoderdata). The entire study design separates accessed from unseen code."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "MinHash + Locality-Sensitive Hashing is used to detect and prevent data leakage between the UNSEEN group and WizardCoder's training set, with 5-grams and a Jaccard similarity threshold of 0.2 (Section III-D1)."
       }
     }
   },
   "claims": [
     {
       "claim": "Even top-performing LLMs produce a non-negligible proportion (0.88% to 2.01%) of code strikingly similar to existing open-source implementations.",
       "evidence": "Table IV shows GPT-4o produces 47 (1.12%), Claude-3.5-Sonnet produces 84 (2.01%), and DeepSeek-Coder-V2 produces 37 (0.88%) strikingly similar cases out of 4,187 benchmark samples.",
       "supported": "strong"
     },
     {
       "claim": "Most LLMs fail to provide accurate license information for code under copyleft licenses, with only Claude-3.5-Sonnet demonstrating some ability.",
       "evidence": "Table IV shows Accc=0.0 for all models except Claude-3.5-Sonnet (Accc=0.4). All other models completely fail on copyleft license identification.",
       "supported": "strong"
     },
     {
       "claim": "The four-criteria striking similarity standard can effectively exclude the possibility of independent creation.",
       "evidence": "In the empirical study (Section III-E), 0/10,000 UNSEEN samples meet the standard while 24/10,000 ACCESSED samples do. Validated on held-out data with WizardCoder (31 cases) and Poro (2 cases), all from ACCESSED_EVAL. Expert validation: 8 reviewers identified 32/33 cases as non-independent creation (Section III-F).",
       "supported": "strong"
     },
     {
       "claim": "Text similarity metrics alone cannot determine non-independent creation in LLM-generated code.",
       "evidence": "Figure 4 shows UNSEEN cases can reach high text similarity (including score of 1.0), demonstrating that text similarity alone is insufficient. The distributions overlap substantially between ACCESSED and UNSEEN groups.",
       "supported": "strong"
     },
     {
       "claim": "Open-source LLMs demonstrate superior compliance performance compared to closed-source LLMs among general models.",
       "evidence": "Table IV shows Qwen2-7B-Instruct (LICO=0.985) and GLM-4-9B-Chat (LICO=1.0) outperform GPT-4o (LICO=0.385) and Claude-3.5-Sonnet (LICO=0.571). However, sample sizes for strikingly similar cases are small and the comparison is confounded by model capabilities.",
       "supported": "weak"
     },
     {
       "claim": "StarCoder2's file-level license detection strategy explains its zero copyleft striking similarity cases.",
       "evidence": "StarCoder2-15B-Instruct shows 0 copyleft strikingly similar cases (Table IV). The paper attributes this to Stack v2's file-level license filtering (Section VI-A1), but this is observational evidence from a single model with no controlled comparison.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Potential feature leakage in license evaluation",
       "detail": "The prompt includes file header comments from code files that contain explicit license information (Section IV-B1 selects files with 'explicit license information in their file header comments'). When the LLM is later asked to provide the license, it may simply recall what was in the prompt context rather than demonstrating genuine license association from training. This potential confound is not discussed."
     },
     {
       "flag": "Very small copyleft sample sizes",
       "detail": "The copyleft license compliance findings are based on extremely small sample sizes: 0–6 copyleft strikingly similar cases per model (Table IV). The claim that 'most LLMs fail' on copyleft is based on models with 0–5 copyleft cases. Statistical conclusions from such small samples are unreliable."
     },
     {
       "flag": "Title overstates scope",
       "detail": "The title says 'Code Generation' broadly but the study only covers Python function-level code completion. Class-level, project-level, and other language code generation are not addressed, acknowledged in threats to validity but not reflected in the title or abstract."
     },
     {
       "flag": "No model version specificity for closed-source models",
       "detail": "Closed-source models (GPT-3.5-Turbo, GPT-4-Turbo, GPT-4o, Claude-3.5-Sonnet, Gemini-1.5-Pro) are specified only by marketing names without API versions or snapshot dates. Model behavior changes across versions, making results non-reproducible."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Pass@k metric, foundational for LLM code generation evaluation."
     },
     {
       "title": "Unveiling memorization in code models",
       "authors": ["Z. Yang", "Z. Zhao", "C. Wang"],
       "year": 2024,
       "relevance": "Studies memorization in LLMs for code, directly related to whether models reproduce training data including licensed code."
     },
     {
       "title": "Traces of memorisation in large language models for code",
       "authors": ["A. Al-Kaswan", "M. Izadi", "A. V. Deursen"],
       "year": 2024,
       "relevance": "Investigates memorization patterns in code LLMs, complementary study on training data reproduction."
     },
     {
       "title": "CodeIPPrompt: intellectual property infringement assessment of code language models",
       "authors": ["Z. Yu", "Y. Wu", "N. Zhang"],
       "year": 2023,
       "relevance": "Most closely related prior work: investigates IP infringement in code LLMs, though under the problematic assumption that LLMs should not generate licensed code at all."
     },
     {
       "title": "Asleep at the keyboard? assessing the security of github copilot's code contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan"],
       "year": 2022,
       "relevance": "Evaluates security of LLM-generated code, a parallel non-functional property assessment for code generation tools."
     },
     {
       "title": "Starcoder: may the source be with you!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Training data and deduplication methodology for StarCoder used as basis for the empirical study in this paper."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["J. Achiam", "S. Adler", "S. Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the major LLM families evaluated in the study."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["N. Carlini", "D. Ippolito", "M. Jagielski"],
       "year": 2022,
       "arxiv_id": "2202.07646",
       "relevance": "Foundational work on measuring memorization in language models, relevant to understanding LLM code reproduction."
     },
     {
       "title": "Do users write more insecure code with ai assistants?",
       "authors": ["N. Perry", "M. Srivastava", "D. Kumar"],
       "year": 2023,
       "relevance": "Studies security implications of AI coding assistants, complementary evaluation of non-functional code properties."
     },
     {
       "title": "WizardCoder: Empowering code large language models with evol-instruct",
       "authors": ["Z. Luo", "C. Xu", "P. Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Core model used in the empirical study for establishing the striking similarity standard."
     }
   ]
 }