ai-research-survey

scan.json (20518B)

---

 {
   "paper": {
     "title": "CORECODEBENCH: Decoupling Code Intelligence via Fine-Grained Repository-Level Tasks",
     "authors": ["Lingyue Fu", "Hao Guan", "Bolun Zhang", "Haowei Yuan", "Yaoming Zhu", "Jun Xu", "Zongyu Wang", "Lin Qiu", "Xunliang Cai", "Xuezhi Cao", "Weiwen Liu", "Weinan Zhang", "Yong Yu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.05281"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided: https://github.com/AGI-Eval-Official/CoreCodeBench and HuggingFace collection for data."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace: https://huggingface.co/collections/tubehhh/corecodebench."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, or detailed environment setup with library versions in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper references appendices for implementation details but does not include a reproducibility guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Confidence intervals and results for additional LLMs are provided in Appendix L' (Section 4.3)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Spearman and Pearson correlations but does not perform significance tests when claiming one model outperforms another. Rankings are compared by raw numbers without statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance differences are reported with baseline context (e.g., 'pass rate disparity exceeding 35%', AC@1 and AC Rate scores with absolute values from which differences can be computed)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 12 repositories were chosen, or why 9 (later 12) LLMs were evaluated. The Multi-BugFix subset has only 10 instances with no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as single-run greedy decoding numbers. No variance across runs or seeds is reported for the main results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 9 state-of-the-art LLMs against each other and correlates with 8 external benchmarks (Section 4.6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-5.2, Claude-4.5-Opus, Gemini-3-Pro, Kimi-K2, DeepSeek-V3.2 — all contemporary frontier models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations: generator backbone sensitivity (Appendix B, Spearman rho=1.0), Information Gain filtering ablation (Appendix C, Table 5), mask length impact (Section 4.5), task quantity impact (Section 4.5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics used: AC@1 (all tests pass) and AC Rate (proportion of fixed test cases), defined in Section 3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Large-scale manual inspection on 360/511 development problems (70.5%) by experienced engineers evaluating readability, accuracy, and completeness, yielding 78.55% qualification rate with >95% inter-annotator agreement (Section 4.2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For fine-tuning validation, a repository-level split (11 train, 1 test) is used to prevent data leakage (Section 4.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by 6 task types (Dev, BugFix, TDD, Multi-Dev, Multi-BugFix, Multi-TDD) in Tables 2 and 3, plus difficulty subsets."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multi-BugFix 0% AC@1 for all models is discussed as a failure mode. The paper analyzes that models 'strictly adhered to the prompt's sequence, ignoring dependency logic' (Section 4.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that all models fail Multi-BugFix (0% AC@1), and discusses the 'performance collapse' and 'critical deficit in hierarchical reasoning' (Section 4.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 78.55% validity yield, capability misalignment, and ranking shifts are all supported by Tables 2-3, Figures 3-4, and Section 4.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are primarily about ablations (mask length causes difficulty increase, IG filtering improves discrimination). These are supported by controlled single-variable manipulations in Sections 4.5 and Appendix C."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Code Intelligence' broadly but results are exclusively on Python repositories. The Limitations section acknowledges this ('CoreCodeBench focuses exclusively on Python') but the title and abstract do not bound the claims to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper considers that capability misalignment could be due to generator bias (Appendix B rules this out with ρ=1.0), and discusses that Multi-BugFix failure may stem from sequential prompting rather than fundamental inability (Section 4.3)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names only: 'GPT-5.2', 'Claude-4.5-Opus', 'Gemini-3-Pro', etc. No API versions, snapshot dates, or specific model IDs are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper references 'detailed implementation settings in Appendix I' and 'prompts in Appendix G' but the extracted text does not include the full prompt text. Prompt templates with placeholders may be in the appendix but the actual fill values are not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'zero-shot setting using greedy decoding' (Section 4.1), which implies temperature=0. Implementation settings referenced in Appendix I."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated in a single-turn zero-shot setting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: repository selection criteria (Section 2.1), context extraction, core code identification with masking, IG filtering (Appendix C retains 48.56%), and validity checking via unit tests."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, discussing Python-only scope, unit test dependency, and Multi-BugFix size."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: dependency on high-coverage unit tests biases toward well-maintained projects, Multi-BugFix has only 10 instances with 'inherently lower statistical power', Python-only scope limits cross-lingual claims."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section explicitly states: 'CoreCodeBench focuses exclusively on Python; extending support to other languages remains a critical direction.' Also acknowledges repositories with sparse tests cannot be processed."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace and code on GitHub, allowing independent verification of the benchmark tasks."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Repository selection criteria (activeness, test coverage >15%, >5000 LOC) detailed in Section 2.1, with full repository metadata in Table 4 (Appendix A)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for the study. The human inspection uses 'experienced engineers' but this is benchmark validation, not a human subjects study. Data source is public PyPI repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented across Sections 2.1-2.3: repository selection → context extraction → atomic task generation → IG filtering (48.56% retained) → composite task generation. Counts provided at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly stated: Shanghai Jiao Tong University and Meituan. Internship relationship disclosed ('Work done while interning at Meituan')."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meituan is a tech company that uses LLMs; some authors are Meituan employees. No funding disclosure makes independence assessment impossible."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the 9 evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses contamination as a core motivation. The pipeline applies 'fine-grained transformations that alter the code's surface form and structural context' to prevent LLMs from 'simply recalling exact solutions from pre-training data' (Section 2.2). The benchmark is designed to be contamination-resilient."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination resilience is a central design goal. The paper argues that code transformations (masking, bug injection, description generation) alter surface form enough to prevent memorization. However, the source repositories (e.g., transformers, langchain) are widely known and likely in training data — the paper's argument is that the tasks are novel transformations of this code."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. The manual inspection is benchmark quality validation, not a human subjects experiment."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API spend, or tokens consumed reported for the evaluations across 9 LLMs on 1,524 tasks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated. The fine-tuning experiment and large-scale evaluations would require significant compute but this is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "CoreCodeBench achieves a 78.55% validity yield, significantly surpassing SWE-bench-Verified's 31.7% retention rate.",
       "evidence": "Section 4.2: Manual inspection of 360/511 development problems by experienced engineers with >95% inter-annotator agreement.",
       "supported": "moderate"
     },
     {
       "claim": "Coding proficiency is non-monolithic: models exhibit significant capability misalignment across cognitive dimensions.",
       "evidence": "Tables 2-3 show distinct ranking shifts across Dev, BugFix, TDD tasks. Figure 4 shows IoU values below 0.7 even for top models. Kimi-K2 pass rate disparity exceeds 35%.",
       "supported": "strong"
     },
     {
       "claim": "All models fail Multi-BugFix (0% AC@1), exposing a critical deficit in hierarchical reasoning.",
       "evidence": "Table 3 shows 0.00% AC@1 for all 9 models on Multi-BugFix. Non-zero AC Rates indicate partial progress.",
       "supported": "strong"
     },
     {
       "claim": "CoreCodeBench captures evaluation dimensions distinct from existing benchmarks.",
       "evidence": "Section 4.6: Hierarchical clustering (Figure 7b) shows CoreCodeBench forms a separate cluster from 8 external benchmarks. Spearman correlations with external benchmarks are generally low.",
       "supported": "moderate"
     },
     {
       "claim": "COREPIPE's evaluation results are invariant to generator backbone choice (Spearman ρ = 1.0).",
       "evidence": "Appendix B: Four generator backbones tested; model rankings remain identical across all configurations.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CoreCodeBench decomposes code intelligence into six task types (Dev, BugFix, TDD, and their multi-function composites) derived from the same code context, revealing that LLM coding proficiency is non-monolithic. Models show significant capability misalignment, with IoU overlap below 0.7 even for top models. All models achieve 0% AC@1 on Multi-BugFix, exposing a ceiling in multi-function debugging. The benchmark's controllable difficulty via mask length and dependency depth provides a mechanism to prevent performance saturation.",
   "red_flags": [
     {
       "flag": "Multi-BugFix extremely small sample",
       "detail": "The Multi-BugFix subset contains only 10 instances, making statistical conclusions about this category unreliable. The paper acknowledges this but still draws conclusions from these results."
     },
     {
       "flag": "No model version specificity",
       "detail": "All models referenced by marketing names only (GPT-5.2, Claude-4.5-Opus, etc.) without API versions or snapshot dates. Results may not be reproducible as model behavior changes across versions."
     },
     {
       "flag": "Single-run greedy evaluation",
       "detail": "All evaluations use single-run greedy decoding with no variance reporting. While greedy decoding is deterministic for a given model version, this prevents assessment of sensitivity to sampling strategies."
     },
     {
       "flag": "Validity comparison with SWE-bench-Verified may be unfair",
       "detail": "The 78.55% vs 31.7% comparison conflates different quality standards and task types. SWE-bench-Verified filters real GitHub issues for solvability, a fundamentally different quality criterion than the readability/accuracy/completeness used here."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational HumanEval benchmark for LLM code generation evaluation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Major repository-level benchmark that CoreCodeBench positions itself against."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2025,
       "arxiv_id": "2406.15877",
       "relevance": "Contemporary code benchmark used in external validity comparison."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-aware code benchmark used in external validity comparison."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu"],
       "year": 2023,
       "relevance": "HumanEvalPlus and MBPP-Plus benchmarks used in external validity analysis."
     },
     {
       "title": "TDD-bench verified: Can LLMs generate tests for issues before they get resolved?",
       "authors": ["Toufique Ahmed"],
       "year": 2024,
       "arxiv_id": "2412.02883",
       "relevance": "Test-driven development benchmark relevant to CoreCodeBench's TDD task type."
     },
     {
       "title": "Kimi K2: Open agentic intelligence",
       "authors": ["Kimi"],
       "year": 2025,
       "arxiv_id": "2507.20534",
       "relevance": "Open-weight agentic LLM evaluated in CoreCodeBench showing large capability misalignment."
     },
     {
       "title": "MultiPL-E: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["Federico Cassano"],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "Multi-language code benchmark used in external validity comparison."
     },
     {
       "title": "On the impacts of contexts on repository-level code generation",
       "authors": ["Nam Le Hai"],
       "year": 2025,
       "arxiv_id": "2406.11927",
       "relevance": "Studies context impact on repository-level code generation, directly relevant to CoreCodeBench's context-based task design."
     },
     {
       "title": "Qwen2.5-coder technical report",
       "authors": ["Binyuan Hui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for Qwen coder models evaluated in the benchmark."
     }
   ]
 }