ai-research-survey

scan.json (21421B)

---

 {
   "paper": {
     "title": "ComplexCodeEval: A Benchmark for Evaluating Large Code Models on More Complex Code",
     "authors": ["Jia Feng", "Jiachen Liu", "Cuiyun Gao", "Chun Yong Chong", "Chaozheng Wang", "Shan Gao", "Xin Xia"],
     "year": 2024,
     "venue": "ASE '24",
     "arxiv_id": "2409.10280",
     "doi": "10.1145/3691620.3695552"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/ComplexCodeEval/ComplexCodeEval (reference [1]). The paper states 'ComplexCodeEval, its construction tools, and all experimental results have been open-sourced.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset is released at the same GitHub repository. The paper explicitly states the benchmark and construction tools are open-sourced."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two A100-80G GPUs' and greedy decoding settings, but no requirements.txt, Dockerfile, or detailed environment specification with library versions is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The experimental setup is described at a high level but lacks specific commands or scripts to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates only (e.g., CodeBLEU scores, F1 scores) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'CodeLlama-34B achieves the highest CodeBLEU score') but uses no statistical significance tests. Comparisons are based solely on raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'incorporating full contextual information increase the average CodeBLEU scores of all LCMs in Java and Python code generation by 70.73% and 31.90%' and absolute deltas in Table 6 for data leakage analysis."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper selects 84-100 samples per task per language (Section 4.5) without justifying why this number is sufficient or conducting a power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation (σ) is reported in Table 3 across models for each metric, showing variance across the different LCMs evaluated."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares performance across 10 LCMs and also compares ComplexCodeEval results against existing benchmarks (CoderEval, CrossCodeEval, Method2Test, APIbench) in Table 8."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include contemporary LCMs: DeepSeek-Coder (2024), StarCoder2 (2024), CodeLlama (2023), and GPT-3.5-Turbo. These were state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 systematically ablates contextual information (D+S, D+S+I, D+S+C, D+S+C+Dep) to measure the contribution of each context type in Tables 4 and 5."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: CodeBLEU, Edit Similarity, BLEU, Exact Match, F1, and Recall across different tasks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is included. All evaluation is automated via CodeBLEU, BLEU, ES, F1, and Recall metrics. Human evaluation was only used for docstring quality during benchmark construction, not for evaluating system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses temporal splits based on model knowledge cutoff dates: 'specifically from data after September 15, 2023, based on the model's knowledge cut-off time' (Section 4.5)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task (code generation, completion, test case generation, API recommendation), by language (Java, Python), by model, and by context condition."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure analysis or specific error examples are discussed. The paper reports aggregate metrics without examining why models fail on particular examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports cases where data leakage did not consistently improve performance (e.g., DSC-33B Python code generation CodeBLEU decreased by 0.21 on leaked data in Table 6), and notes inconsistent patterns across models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about LCM limitations, context impact, and data leakage effects are all supported by experimental results in Sections 5.1-5.3 with specific numbers matching the abstract's claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about context improving performance, tested via controlled ablation (adding context incrementally). The data leakage analysis uses temporal splits as a quasi-experimental design. These constitute adequate controlled single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims evaluation of 'Large Code Models' generally, but testing is limited to Java and Python with 10 specific models. The paper does not explicitly bound generalizations to these languages and models in the abstract or conclusion."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for observed performance differences. For example, the data leakage analysis does not consider that code from different time periods may differ in complexity or style independent of leakage."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are named by family and size (e.g., 'DeepSeek-Coder-33B', 'GPT-3.5-Turbo') but no specific version identifiers or snapshot dates are provided for the API models. Table 2 lists training cutoff dates but not model version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The docstring generation prompt template is shown in Figure 4, but the actual prompts used for the four evaluation tasks (code generation, completion, test case generation, API recommendation) are not provided. The paper describes task inputs at a high level without showing exact prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 states: 'we set the temperature to 0, top_p to 1, and max_tokens to 1000 (with the API-recommended setting being 10).'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated via direct prompting for each task."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full data collection and preprocessing pipeline in detail: library selection via SourceRank, repository filtering by stars (>99), AST-based API extraction, test case matching, deduplication, and docstring generation. Specific numbers are provided at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 'Threats To Validity' discusses threats in benchmark construction and empirical study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses specific threats: timestamp fallback mechanism for missing git commit data, and computing resource constraints limiting model selection. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. It acknowledges limited model coverage but does not bound generalizations about what languages, domains, or model types are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset and experimental results are open-sourced at the GitHub repository (reference [1])."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides detailed data collection procedures: library selection from Libraries.io, repository selection from GitHub (>99 stars), AST-based API extraction, with specific criteria and counts at each stage."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is GitHub repositories selected by objective criteria (star count, library dependencies)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section 3 with counts: 85,000+ repositories → 9,169 Java and 27,178 Python after dependency filtering → 3,897 Java and 7,184 Python samples after extraction and deduplication."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section lists funding: National Natural Science Foundation of China (No. 62472126), Natural Science Foundation of Guangdong Province, Shenzhen-Hong Kong Jointly Funded Project, and others."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: three authors from Huawei (Chun Yong Chong, Shan Gao, Xin Xia), others from universities. This is relevant since no Huawei products are directly evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from government research foundations (NSFC, Guangdong Province, Shenzhen). These funders have no stake in which LCM performs better on the benchmark."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Three authors are affiliated with Huawei, but no disclosure statement addresses potential conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 lists training data cutoff dates for all models: DeepSeek-Coder and CodeLlama at 2023-01-01, StarCoder2 at 2023-09-14, GPT-3.5-Turbo at 2021-10-01."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Section 5.3) is entirely dedicated to analyzing the impact of data leakage. The paper uses temporal splits and Exact Match metrics to detect and quantify train/test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark deliberately includes timestamps for each sample to enable contamination-aware evaluation. Section 3.2.4 describes the time-tagging approach, and RQ3 directly measures the effect of contamination on performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The docstring quality check by two authors is part of benchmark construction, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or wall-clock times are reported despite using GPT-3.5-Turbo (a paid API) and running experiments on A100 GPUs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (two A100-80G GPUs) but total GPU hours, training time, or API spend are not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "LCMs exhibit suboptimal performance in complex development scenarios, with the best CodeBLEU scores of 27.54 (Python) and 34.08 (Java) for code generation.",
       "evidence": "Table 3 shows performance of 10 LCMs across 4 tasks. CodeLlama-34B achieves highest CodeBLEU in code generation (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Rich contextual information greatly enhances LCM performance, with full context increasing average Java code generation CodeBLEU by 70.73%.",
       "evidence": "Table 4 shows systematic context ablation (D+S → D+S+I → D+S+C → D+S+C+Dep) with consistent improvements across all models (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "Data leakage leads to overestimation of LCM performance, with average CodeBLEU increases of 1.22 (Java) and 3.10 (Python) on leaked vs. non-leaked data.",
       "evidence": "Table 6 compares performance on pre-cutoff vs. post-cutoff data with Exact Match metrics confirming leakage presence (Section 5.3).",
       "supported": "moderate"
     },
     {
       "claim": "LLM-generated docstrings improve code generation performance, with average CodeBLEU increasing by 3.33 compared to original docstrings.",
       "evidence": "Table 7 compares original vs. generated docstrings across 9 models in Python code generation (Section 6.1).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ComplexCodeEval introduces a multi-task benchmark with 11,081 samples from 3,162 GitHub repositories covering code generation, completion, test case generation, and API recommendation in Java and Python. The benchmark incorporates timestamps to detect and mitigate data leakage. Key findings show that LCMs still struggle with complex code (best CodeBLEU ~34 for Java code generation), that adding contextual information dramatically improves performance (up to 70.73% improvement), and that data leakage inflates evaluation scores by 1-7 points depending on task and model.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims between models and conditions are based on raw metric differences without any significance testing. With only 84-100 samples per condition, observed differences could be due to chance."
     },
     {
       "flag": "Small evaluation sample size",
       "detail": "Only 84-100 samples per task per language are used for evaluation (Section 4.5), which is small for making broad claims about LCM capabilities. No justification for this sample size is provided."
     },
     {
       "flag": "Huawei author affiliations without conflict disclosure",
       "detail": "Three of seven authors are affiliated with Huawei, but no competing interests statement is provided. While no Huawei products are directly evaluated, the absence of a disclosure statement is a gap."
     },
     {
       "flag": "Variance metric is across models not runs",
       "detail": "The σ column in Table 3 appears to measure standard deviation across different models, not across multiple runs of the same model. Since greedy decoding is used (temperature=0), there is only one run per model, so variance across experimental runs is not assessed."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a foundational benchmark for code generation evaluation that ComplexCodeEval aims to improve upon."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for evaluating LLMs on code generation tasks."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming–The Rise of Code Intelligence",
       "authors": ["Daya Guo"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the main LCM families evaluated in ComplexCodeEval."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the main LCM families evaluated, achieving best performance on several tasks."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "authors": ["Anton Lozhkov"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "One of the main open-source LCM families evaluated in the benchmark."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Related contamination-free benchmark using temporal updates, which ComplexCodeEval's timestamp approach is inspired by."
     },
     {
       "title": "CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion",
       "authors": ["Yangruibo Ding"],
       "year": 2023,
       "relevance": "Cross-file code completion benchmark used as comparison point in the evaluation."
     },
     {
       "title": "Codereval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["Hao Yu"],
       "year": 2024,
       "relevance": "Repository-level code generation benchmark used as direct comparison in Table 8."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Class-level code generation benchmark compared in Table 1 for benchmark characteristics."
     },
     {
       "title": "EvoCodeBench: An Evolving Code Generation Benchmark Aligned with Real-World Code Repositories",
       "authors": ["Jia Li"],
       "year": 2024,
       "arxiv_id": "2404.00599",
       "relevance": "Repository-level evolving benchmark addressing data leakage, closely related to ComplexCodeEval's goals."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li"],
       "year": 2022,
       "relevance": "Large-scale competitive code generation system relevant to LCM capability evaluation."
     }
   ]
 }