ai-research-survey

scan.json (27615B)

---

 {
   "paper": {
     "title": "Challenge on Optimization of Context Collection for Code Completion",
     "authors": [
       "Dmitry Ustalov",
       "Egor Bogomolov",
       "Alexander Bezzubov",
       "Yaroslav Golubev",
       "Evgeniy Glukhov",
       "Georgii Levtsov",
       "Vladimir Kovalenko"
     ],
     "year": 2025,
     "venue": "ASE 2025 Workshops (40th IEEE/ACM International Conference on Automated Software Engineering)",
     "arxiv_id": "2510.04349"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A starter kit with baseline solutions is released on GitHub (https://github.com/JetBrains-Research/ase2025-starter-kit), referenced in Section III. The private phase submissions from competing teams are also included in the Zenodo dataset."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C states: 'The complete competition dataset, including ground truth data, repositories, and private phase submissions, can be downloaded from Zenodo [18] under the CC BY 4.0 license.' A specific Zenodo URL is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, Dockerfile, conda environment, or dependency versions) is described in the paper for reproducing the evaluation pipeline. The starter kit may contain this, but the paper itself does not document it."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the competition workflow is described at a high level (Section II.A), there are no step-by-step reproduction instructions for rerunning the evaluation pipeline. The paper describes the conceptual workflow but not concrete commands or scripts to replicate the results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables II-V report only point estimates of chrF scores. No confidence intervals, error bars, or uncertainty measures are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Rankings are determined solely by comparing raw chrF averages. No statistical significance tests are used to determine whether differences between teams are statistically meaningful. For example, in Table V, teams WSPR NCSU and REALISE Lab share rank 3 due to 'negligible difference' (0.660 vs 0.659), but this is judged informally rather than through a statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw chrF scores are reported. No effect sizes, percentage improvements over baselines with context, or standardized measures of difference magnitude are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset sizes (688 Python completion points, 1,076 Kotlin completion points) are stated but not justified. No power analysis or reasoning for why these sizes are sufficient for reliable ranking is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are single-run evaluations. No variance, standard deviation, or spread measures across multiple runs or completion points are reported. Only aggregate averages across all completion points and three models are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section II.B describes several baselines: empty context, random recent file, most similar file according to BM25, and recent files strategy. These appear in Tables II-V for comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines (BM25, recent files) represent common practical strategies used in modern IDEs, as stated in Section II.B: 'Based on our experience, most IDEs on the market use the recent files strategy.' These are appropriate for the competition setting."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a competition report, not a system paper. The organizers did not propose a single system to ablate. Individual team papers may contain ablations, but this overview paper describes the competition structure and results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only chrF is used as the evaluation metric. Section II.B justifies this choice by citing Evtikhiev et al. [14], but no additional metrics (e.g., exact match, BLEU, CodeBLEU, pass@k) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of completion quality is included. All evaluation is automated via the chrF metric. Human judgment of code completion quality could provide complementary insights but was not conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section II.A describes a clear separation: practice, public, and private phases with non-overlapping data. Section II.C confirms: 'no repository appeared in more than one subset of the data (practice, public, private).' Final rankings used the private held-out dataset."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language (Python vs Kotlin) and by model (Mellum, Codestral, Qwen chrF scores reported separately in all tables). This provides meaningful per-category information."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is presented. The paper does not discuss which types of completion points were hardest, where solutions failed, or qualitative examples of poor completions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. The paper does not discuss approaches that failed to beat baselines (besides listing rankings), configurations that were tried and abandoned by the organizers, or design decisions that did not work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims are modest and factual: it describes the competition structure, number of participants (19 Python, 8 Kotlin public; 6 private), and that solutions were evaluated using chrF. These are all supported by the detailed content in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section I states 'Context quality is so critical that a smaller model with better context clues can outperform a larger, more capable model [8]' -- a causal claim attributed to cited work but used to motivate the competition. More importantly, the paper implicitly treats chrF differences as reflecting context quality differences, but confounds between team skill, implementation details, and context strategy are not disentangled."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states the competition 'allowed us to explore the solution space and identify the most practical strategies for context collection' -- this generalizes beyond the specific dataset, models, and metric tested. The paper does not bound its conclusions to the specific Python/Kotlin datasets, three models, and chrF metric used."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for performance differences are discussed. For example, performance differences could stem from engineering quality rather than context strategy design, or from overfitting to specific models during the public phase. Section IV.G mentions data leakage but does not discuss other confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The three evaluation models are named as 'Codestral by Mistral AI [10]', 'Qwen2.5-Coder [11]', and 'Mellum [12]'. No specific version identifiers, snapshot dates, or model sizes are given for Codestral or Qwen2.5-Coder. Mellum links to a blog post but no specific checkpoint is identified for the evaluation."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section II.A describes that the platform 'converted each context into a model-specific prompt... by inserting the model-specific special tokens and arranging context, prefix, and suffix in the format required by the model.' The actual prompt templates and special tokens used for each model are not provided in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference hyperparameters (temperature, top-p, max tokens, beam size) are reported for the three completion models. These can significantly affect code completion quality but are not disclosed."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a straightforward pipeline: context collection -> prompt construction -> model inference -> metric computation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C describes the dataset creation process: repositories were collected from permissively-licensed GitHub projects, commit histories were enumerated, multi-line insertions were identified using content-based heuristics, and data was split into practice/public/private subsets with no repository overlap. Sizes are given for each split."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.G 'Threats to Validity' is present and discusses a specific concern about data leakage from solutions using all available repository data including future revisions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.G identifies a specific threat: 'the solutions by teams SpareCodeComplete and NoMoreActimel use all available data in the provided dataset, including other repositories and all the revisions for the given repository. Such an approach may lead to a data leakage from the future versions of the same repository.' This is specific to this competition."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. For example, it does not acknowledge that results are limited to: only two programming languages, only fill-in-the-middle completion, only three specific models, only the chrF metric, or only permissively licensed open-source code. No explicit scope boundaries or 'what we do not claim' section is present."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The complete competition dataset including ground truth, repositories, and private phase submissions is available on Zenodo [18] under CC BY 4.0 license, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C describes the collection procedure: built on Long Code Arena [6], collected from permissively-licensed open-source GitHub repositories, enumerated commit histories, retained commits with significant multi-line insertions, generated multi-line splits using in-house code analysis tools."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research sense. The competition participants are not research subjects. Data comes from public GitHub repositories, making this NA."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C documents the pipeline: selection of large permissively-licensed repositories -> commit history enumeration -> filtering for multi-line insertions -> split generation using code analysis tools -> separation into practice/public/private subsets. Final counts are provided (102 repositories, 1,176 revisions, 1,764 completion points)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding disclosure is present. The Acknowledgments section thanks colleagues at JetBrains and Mistral AI but does not state funding sources or grants. Mistral AI 'offset' Codestral API expenses, but formal funding is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors' JetBrains affiliations are clearly stated in the author block. JetBrains organized the competition and one of the three evaluation models (Mellum) is JetBrains' own product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "JetBrains organized and funded the competition, and one of the three evaluation models (Mellum) is JetBrains' product. Mistral AI co-sponsored and provided Codestral. Both organizations have a commercial interest in demonstrating that context collection improves their models' performance. The funders are not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The authors are JetBrains employees evaluating a competition that includes JetBrains' own model, but no conflict of interest declaration is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Three pre-trained models (Codestral, Qwen2.5-Coder, Mellum) are used to evaluate code completions on public GitHub repositories. No training data cutoff dates are stated for any of the models. The dataset is constructed from public repos that could have been in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the GitHub repositories in the competition dataset might have appeared in the training data of Codestral, Qwen2.5-Coder, or Mellum. Section II.C mentions temporal separation via commit histories but only for context-to-target leakage within the competition, not for model training data contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The dataset is derived from public GitHub repositories. All three models were likely trained on public code. The paper does not address whether the specific repositories or code patterns in the dataset appeared in model training data. Section IV.G discusses data leakage within solutions but not model training contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research sense. Competition participants are not human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research sense. Competition participants are not human subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research sense. Competition participants are not human subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research sense. Competition participants are not human subjects."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research sense. This is not an experimental study with randomized conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research sense. This is not an experimental study requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research sense. Competition participants are not human subjects."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The competition required running three neural models across thousands of completion points. No API costs, compute costs, or inference time per completion are reported. Section III mentions 'API keys at our expense' but does not quantify costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report hardware used for evaluation, total API spend, GPU hours for running the models, or wall-clock time for the evaluation process."
       }
     }
   },
   "claims": [
     {
       "claim": "Context collection strategy significantly affects code completion quality, to the extent that a smaller model with better context can outperform a larger model.",
       "evidence": "Cited as reference [8] in Section I. Not directly demonstrated in this paper's results, but the competition results show substantial variation in chrF scores across different context strategies (e.g., Python private phase ranges from 0.610 to 0.734 average chrF, Tables IV-V).",
       "supported": "moderate"
     },
     {
       "claim": "Nineteen teams submitted solutions to the Python track and eight teams submitted solutions to the Kotlin track during the public phase.",
       "evidence": "Stated in the abstract and Section III, confirmed by the leaderboard tables (Tables II and III).",
       "supported": "strong"
     },
     {
       "claim": "Most competitive strategies share common traits: using parsing tools to extract definitions and symbols, applying classical IR ranking (e.g., BM25), and assembling context within token budgets.",
       "evidence": "Section IV describes six private-phase solutions. Five of six use some form of symbol extraction (AST, Tree-sitter, PSI), and four of six use BM25 or similar retrieval. Summarized in the opening of Section IV.",
       "supported": "moderate"
     },
     {
       "claim": "The competition dataset is the first to provide multi-line fill-in-the-middle completion with separate Python and Kotlin tracks from permissively licensed code.",
       "evidence": "Section II.C describes differences from Long Code Arena: 'it provides multi-line completion using a fill-in-the-middle approach rather than prefix-based completion, it includes subsets for Python and Kotlin programming languages.' No explicit claim of being 'first' but implicitly positioned as novel.",
       "supported": "moderate"
     },
     {
       "claim": "Some solutions may suffer from data leakage because they use all available repository data including future revisions.",
       "evidence": "Section IV.G identifies that teams SpareCodeComplete and NoMoreActimel 'use all available data in the provided dataset, including other repositories and all the revisions for the given repository' which 'may lead to a data leakage from the future versions of the same repository.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "This paper describes a community competition at ASE 2025 on optimizing context collection for fill-in-the-middle code completion in Python and Kotlin. The winning solutions combined code parsing (AST, Tree-sitter) with information retrieval (BM25, FAISS) to select relevant context snippets from repositories. The top team (NoMoreActimel for Python, SpareCodeComplete for Kotlin) substantially outperformed baselines, with average chrF scores of 0.734 and 0.748 respectively on the private test sets. The competition dataset of 1,764 completion points from 102 repositories has been released on Zenodo.",
   "red_flags": [
     {
       "flag": "Organizer conflict of interest",
       "detail": "All authors are JetBrains employees. One of the three evaluation models (Mellum) is JetBrains' own product, and Mistral AI (co-sponsor) provided Codestral. The competition design, metric selection, and dataset construction were all controlled by parties with commercial interest in the evaluated models. No conflict of interest statement is provided."
     },
     {
       "flag": "No statistical analysis of ranking differences",
       "detail": "Team rankings are determined solely by raw chrF averages with no statistical tests. Some score differences are very small (e.g., WSPR NCSU 0.660 vs REALISE Lab 0.659 on Kotlin private) and it is unclear whether rankings reflect meaningful performance differences or noise."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only chrF is used as the evaluation metric. Code completion quality is multidimensional (syntactic correctness, semantic correctness, usefulness to developers), and chrF captures only character-level similarity. This limits the conclusions that can be drawn about which context strategies are truly better."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "The dataset is derived from public GitHub repositories. All three evaluation models were likely trained on public code. The paper addresses within-competition temporal leakage but does not discuss whether the models may have seen the evaluation code during pre-training."
     },
     {
       "flag": "Acknowledged data leakage risk in winning solutions",
       "detail": "Section IV.G notes that the 1st and 2nd place teams (NoMoreActimel and SpareCodeComplete) used all available repository data including future revisions, which 'may lead to a data leakage.' These are the prize-winning solutions, yet this validity threat is noted but not investigated."
     }
   ],
   "cited_papers": [
     {
       "title": "CodeFill: Multi-Token Code Completion by Jointly Learning from Structure and Naming Sequences",
       "authors": ["M. Izadi", "R. Gismondi", "G. Gousios"],
       "year": 2022,
       "relevance": "Code completion technique using neural networks with structural information, relevant to AI-assisted programming."
     },
     {
       "title": "Efficient Training of Language Models to Fill in the Middle",
       "authors": ["M. Bavarian", "H. Jun", "N. Tezak", "J. Schulman", "C. McLeavey", "J. Tworek", "M. Chen"],
       "year": 2022,
       "arxiv_id": "2207.14255",
       "relevance": "Foundational work on fill-in-the-middle training for LLMs, the core paradigm evaluated in this competition."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "authors": ["F. Zhang", "B. Chen", "Y. Zhang", "J. Keung", "J. Liu", "D. Zan", "Y. Mao", "J.-G. Lou", "W. Chen"],
       "year": 2023,
       "relevance": "Repository-level code completion using retrieval, directly relevant to the context collection strategies evaluated in this competition."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code LLM family relevant to understanding model capabilities for code completion."
     },
     {
       "title": "Repoformer: Selective Retrieval for Repository-Level Code Completion",
       "authors": ["D. Wu", "W. U. Ahmad", "D. Zhang", "M. K. Ramanathan", "X. Ma"],
       "year": 2024,
       "relevance": "Selective retrieval approach for repository-level completion, closely related to context optimization strategies."
     },
     {
       "title": "Long Code Arena: A Set of Benchmarks for Long-Context Code Models",
       "authors": ["E. Bogomolov", "A. Eliseeva", "T. Galimzyanov", "E. Glukhov"],
       "year": 2024,
       "arxiv_id": "2406.11612",
       "relevance": "The benchmark this competition dataset builds upon, evaluating long-context code models."
     },
     {
       "title": "RLCoder: Reinforcement Learning for Repository-Level Code Completion",
       "authors": ["Y. Wang", "Y. Wang", "D. Guo", "J. Chen", "R. Zhang", "Y. Ma", "Z. Zheng"],
       "year": 2025,
       "relevance": "RL-based approach to context selection for repository-level code completion, demonstrating that context quality can outweigh model size."
     },
     {
       "title": "Repohyper: Search-Expand-Refine on Semantic Graphs for Repository-Level Code Completion",
       "authors": ["H. N. Phan", "H. N. Phan", "T. N. Nguyen", "N. D. Bui"],
       "year": 2025,
       "relevance": "Semantic graph-based approach to repository-level code completion, relevant to context collection optimization."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "authors": ["B. Hui", "J. Yang", "Z. Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for one of the three evaluation models used in the competition."
     },
     {
       "title": "Out of the BLEU: How Should We Assess Quality of the Code Generation Models?",
       "authors": ["M. Evtikhiev", "E. Bogomolov", "Y. Sokolov", "T. Bryksin"],
       "year": 2023,
       "relevance": "Study on code generation evaluation metrics that motivated the choice of chrF in this competition."
     },
     {
       "title": "On the Importance of Context Filtering in Retrieval-Augmented Code Completion",
       "authors": ["S. Sedov", "V. Savinskiy", "A. Arzhantsev"],
       "year": 2025,
       "relevance": "Paper by the winning Python team (NoMoreActimel) describing their retrieval-augmented context collection strategy."
     },
     {
       "title": "SpareCodeSearch: Searching for Code Context When You Have No Spare GPU",
       "authors": ["M. Nguyen"],
       "year": 2025,
       "relevance": "Paper by the winning Kotlin team (SpareCodeComplete) describing their trigram-index-based context search approach."
     }
   ]
 }