ai-research-survey

scan.json (22644B)

---

 {
   "paper": {
     "title": "Control Models for In-IDE Code Completion: Saving Inference Costs While Improving Completion Quality Metrics",
     "authors": ["Aral de Moor", "Yana Hrynevich", "Hleb Badzeika", "Vladyslav Furda", "Marko Kojic", "Artem Savelev", "Kostadin Cvejoski", "Darya Rovdo", "Ekaterina Garanina"],
     "year": 2026,
     "venue": "IDE '26 (3rd International Workshop on Integrated Development Environments)",
     "arxiv_id": "2601.20223",
     "doi": "10.1145/3786151.3788608"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The paper mentions an 'open-source development of our ML API' in acknowledgments but provides no link to code for the control models themselves."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The datasets consist of anonymised telemetry logs from JetBrains IDEs. No dataset is released or linked. Section 3.2 describes the data but provides no download link."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or library versions are provided. CatBoost is mentioned as the boosting framework but no version is given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The paper describes an approach using proprietary telemetry data with no guidance on how to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports relative change percentages with statistical significance markers but no confidence intervals or error bars. Figures show curves without uncertainty bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states 'significance is computed using bootstrap resampling with individual users as the sampling unit' at p≤0.05. Non-significant results are marked with asterisks in Table 2."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports relative changes with baselines implicit (e.g., Filter model: +46.5% AR for Java, -9.6% RoCC). The percentage deltas with context of what is being measured provide effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are reported (e.g., 151/127 users for Java filter A/B) but no power analysis or justification for these sizes is given. Some A/B groups are quite small (95/62 for Kotlin filter)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for any results. Table 2 shows only point estimates of relative changes."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The A/B study compares with and without control models (Table 2). Offline evaluation compares boosting vs. transformer architectures (Section 4.1, Figure 2). The no-model condition serves as baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons are against their own production system and against the transformer approach from de Moor et al. 2024 and Sun et al. 2025, which are recent related works."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 ablates trigger and filter models separately in A/B tests. Section 4.1 investigates the interplay between trigger and filter at different filter rates and FNR thresholds (Figures 2-3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: Ratio of Completed Code (RoCC), Accept Rate (AR), and Cancel Rate (CR), defined in Section 3.3. Generations count is also reported for trigger models."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates via automated metrics (accept/cancel rates). No human evaluation of output quality or user satisfaction surveys are included, despite claims about user perception and productivity."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.2 describes disjoint user splits: '126 users in the train split... and 98 users in the test split' with guaranteed no user overlap. The A/B study is a separate online evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by language (Kotlin, Python, PHP, C# in Figures 3a-3c) and by model type (trigger vs. filter in Table 2). Per-language A/B results are shown for Java, Python, Kotlin."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses the unexpected PHP result where trigger outperforms filter. Section 4.2 notes the Kotlin Cancel Rate abnormality. Section 5.1 discusses the gap between offline and online metrics."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that filter models reduce RoCC (Table 2: -9.6% to -14.0%), that transformer models don't improve accept/cancel rates as well as boosting (Section 4.1), and that the trigger model didn't significantly reduce generations as expected (Section 4.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims control models 'save around 20% of inference requests while improving completion quality metrics.' Table 2 shows -13.8% generations for trigger, and AR/CR improvements. The 20% is discussed in Section 4.3 as the offline threshold, with 13.8% realized online."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about control models improving metrics are supported by A/B testing (Section 4.3), which is an appropriate experimental design for causal inference. Users are the randomization unit."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly limits scope to 'JetBrains IDEs' (Section 2.2), notes 'While this study is likely relevant to any similar tool, our focus is on the cloud-based code completion provided by JetBrains.' The title specifies 'In-IDE Code Completion.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses how completion event dependencies explain the gap between offline and online metrics. Section 4.2 hypothesizes about the Kotlin Cancel Rate abnormality. Section 4.3 discusses how hard rules may inflate filter model metrics."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '100M-parameter code generation model' as the transformer backbone and CatBoost for boosting, but provides no specific version numbers for any model or library used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. The control models are ML classifiers (CatBoost boosting and transformer classifiers), not prompt-based systems."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for either the CatBoost models or the transformer models. No learning rates, tree depths, number of estimators, or training configurations are provided."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The control models are classifiers integrated into the IDE completion pipeline, not agentic systems."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.2 describes data sources at a high level (telemetry logs, 'several hundred features') but does not document preprocessing, filtering, or transformation steps. Feature engineering details are absent."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 ('Discussion on Metrics') and Section 6 ('Future Work') discuss limitations including offline-to-online metric gaps (5.1), long-term productivity measurement challenges (5.2), and transformer deployment obstacles (6.1)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses the specific threat of completion event dependence affecting metric validity. Section 3.2.1 notes that training data requires disabling models, creating data collection constraints. Section 4.3 notes hard rules inflating filter metrics."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly states 'We limit the scope of this paper to LLM-driven code completion in JetBrains IDEs.' Section 5.2 acknowledges that current metrics measure perceived rather than actual productivity, citing contrary evidence."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is released. The data consists of proprietary anonymised telemetry from JetBrains IDE users."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes data as 'anonymised feature usage logs from Mellum cloud-inferenced completion,' with details on feature types (Section 3.2.1), code context collection (Section 3.2.2), and dataset sizes (Table 1)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how users were selected for the telemetry dataset or the A/B study. Section 3.2.2 mentions 'internal users' for code context but does not describe recruitment or selection criteria."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw telemetry to training data is not documented. No filtering criteria, data cleaning steps, or counts of removed samples are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement is present. All authors are affiliated with JetBrains but no explicit funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All nine authors list JetBrains or JetBrains Research as their affiliation, clearly visible in the author list."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors work at JetBrains, which directly benefits from demonstrating that control models improve its IDE products. The funder (JetBrains as employer) has a financial interest in positive results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. The authors are employed by the company whose product they evaluate."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates ML classifiers (CatBoost, transformer) on telemetry data, not a pre-trained LLM's capability on a benchmark. Contamination in the benchmark sense does not apply."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of a pre-trained model. However, the paper does address train/test user separation (Section 3.2.2), which is good practice for their setting."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not evaluate a pre-trained model on a public benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The A/B study involves human users. No pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The A/B study deploys experimental conditions to production users. No IRB or ethics approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No demographics of the A/B study participants or telemetry dataset users are reported beyond counts (e.g., 151/127 users). Experience level, geographic distribution, etc. are absent."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion/exclusion criteria for A/B study participants are stated. It is unclear how users were selected for the experiment."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'A/B study' and 'A/B groups' but does not describe the randomization procedure for assigning users to conditions."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether users knew they were in an experiment or which condition they were assigned to."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on participant attrition or dropout during the A/B study is provided."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper's goal is to reduce inference costs but does not report actual cost figures (dollars, latency numbers) for the control models or the LLM completions they gate."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget, training time, or hardware specifications are provided for training the control models."
       }
     }
   },
   "claims": [
     {
       "claim": "Boosting-based control models can save around 20% of inference requests while improving completion quality metrics.",
       "evidence": "Table 2 shows the trigger model reduces generations by 13.8% online (20% offline threshold). Filter models improve AR by +32.9% to +47.6% and reduce CR by -15.5% to -36.7%, but decrease RoCC by -9.6% to -14.0%.",
       "supported": "moderate"
     },
     {
       "claim": "Transformer-based models can filter out more generations at lower false-negative cost than boosting models.",
       "evidence": "Figure 2a shows transformer models achieve higher filter rates for the same Symbols Completed hit at certain operating points (Section 4.1).",
       "supported": "moderate"
     },
     {
       "claim": "Transformer-based models do not improve Accept and Cancel Rates as well as boosting models despite filtering more.",
       "evidence": "Section 4.1 and Figures 2b-2c show transformers affect rate-based metrics less favorably because 'the denominator of these rate-based metrics (shown completions) is relatively less affected.'",
       "supported": "moderate"
     },
     {
       "claim": "The trigger model did not reduce online generations as much as the offline threshold suggested (13.8% vs 20%).",
       "evidence": "Section 4.3 and 5.1 explain this through completion event dependencies: 'by affecting completion behaviour, users change their interactions with code completions, resulting in more overall opportunities for completions.'",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "The paper introduces 'control models' — lightweight ML classifiers (CatBoost boosting and transformers) that gate when LLM code completions are triggered and shown in JetBrains IDEs. Offline evaluation on 98 users shows transformer models can filter more completions with lower false-negative rates, but boosting models better improve accept/cancel rate metrics. An online A/B study demonstrates that boosting-based filter models improve accept rates by 33-48% and reduce cancel rates by 16-37%, at the cost of 10-14% reduction in ratio of completed code. A key finding is that offline metrics do not directly translate to online due to dependencies between completion events.",
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "All nine authors are JetBrains employees evaluating JetBrains IDE code completion. No external validation or independent evaluation is included."
     },
     {
       "flag": "Small A/B sample sizes without justification",
       "detail": "Filter model A/B groups range from 62 to 151 users per condition. Several key metrics (RoCC for all filter languages, CR for Python and Kotlin) are not statistically significant, suggesting the studies may be underpowered."
     },
     {
       "flag": "No reproducibility artifacts",
       "detail": "No code, data, environment specifications, or reproduction instructions are provided. The proprietary telemetry data makes independent replication impossible."
     },
     {
       "flag": "Missing hyperparameters and training details",
       "detail": "No hyperparameters, training configurations, or model details beyond architecture type are provided, making it impossible to assess whether the models were properly tuned."
     }
   ],
   "cited_papers": [
     {
       "title": "Grounded Copilot: How Programmers Interact with Code-Generating Models",
       "authors": ["Shraddha Barke", "Michael James", "Nadia Polikarpova"],
       "year": 2022,
       "doi": "10.1145/3586030",
       "relevance": "Foundational study on developer interaction modes (accelerative vs. explorative) with AI code completion."
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "doi": "10.48550/arXiv.2507.09089",
       "relevance": "RCT finding misalignment between perceived (+20%) and actual (-20%) productivity with AI coding tools."
     },
     {
       "title": "When to Show a Suggestion? Integrating Human Feedback in AI-Assisted Programming",
       "authors": ["Hussein Mozannar", "Gagan Bansal", "Adam Fourney", "Eric Horvitz"],
       "year": 2024,
       "doi": "10.1609/aaai.v38i9.28878",
       "relevance": "GitHub-affiliated work on trigger and filter models for code completion, reporting 13% trigger savings and 25% filter rate."
     },
     {
       "title": "Don't Complete It! Preventing Unhelpful Code Completion for Productive and Sustainable Neural Code Completion Systems",
       "authors": ["Zhensu Sun", "Xiaoning Du", "Fu Song"],
       "year": 2025,
       "doi": "10.1145/3688831",
       "relevance": "Transformer-based trigger model rejecting 20% of completion requests with 97.4% precision on unhelpful completions."
     },
     {
       "title": "A Transformer-Based Approach for Smart Invocation of Automatic Code Completion",
       "authors": ["Aral de Moor", "Arie van Deursen", "Maliheh Izadi"],
       "year": 2024,
       "doi": "10.1145/3664646.3664760",
       "relevance": "Prior work by first author on hybrid transformer control models combining telemetry with code context."
     },
     {
       "title": "Mellum: Production-Grade in-IDE Contextual Code Completion with Multi-File Project Understanding",
       "authors": ["Nikita Pavlichenko", "Iurii Nazarov", "Ivan Dolgov"],
       "year": 2025,
       "arxiv_id": "2510.05788",
       "relevance": "JetBrains' production code completion system that the control models gate."
     },
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["Albert Ziegler", "Eirini Kalliamvakou"],
       "year": 2022,
       "doi": "10.1145/3520312.3534864",
       "relevance": "GitHub Copilot productivity assessment showing acceptance rate correlates with perceived productivity."
     },
     {
       "title": "Full Line Code Completion: Bringing AI to Desktop",
       "authors": ["Anton Semenkin", "Vitaliy Bibaev"],
       "year": 2024,
       "doi": "10.1109/ICSE-SEIP66354.2025.00055",
       "relevance": "JetBrains local code completion system describing the RoCC metric and hard filtering rules."
     },
     {
       "title": "Reading Between the Lines: Modeling User Behavior and Costs in AI-Assisted Programming",
       "authors": ["Hussein Mozannar", "Gagan Bansal", "Adam Fourney", "Eric Horvitz"],
       "year": 2024,
       "doi": "10.1145/3613904.3641936",
       "relevance": "Models user behavior and costs during AI-assisted programming interactions."
     },
     {
       "title": "\"It's Weird That it Knows What I Want\": Usability and Interactions with Copilot for Novice Programmers",
       "authors": ["James Prather", "Brent N. Reeves", "Paul Denny"],
       "year": 2023,
       "doi": "10.1145/3617367",
       "relevance": "Study of novice interactions with code completion, identifying shepherding and drifting modes."
     }
   ]
 }