ai-research-survey

scan.json (27672B)

---

 {
   "paper": {
     "title": "EditFlow: Benchmarking and Optimizing Code Edit Recommendation Systems via Reconstruction of Developer Flows",
     "authors": ["Chenyan Liu", "Yun Lin", "Jiaxin Chang", "Jiawei Liu", "Binhang Qi", "Bo Jiang", "Zhiyong Huang", "Jin Song Dong"],
     "year": 2026,
     "venue": "OOPSLA (Proc. ACM Program. Lang.)",
     "arxiv_id": "2602.21697",
     "doi": "10.1145/3798249"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides an anonymous website [3] (sites.google.com/view/editflow) with source code, auto-tuned prompt, dataset, and experiment results. They also implement a VS Code extension."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The annotated dataset of 100 commits with edit order labels is released via their website. The industrial dataset cannot be released due to compliance restrictions, but the annotated benchmark is available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, or detailed environment/dependency specifications in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The website is referenced but the paper itself does not include specific commands or a reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 5-7 report point estimates only (percentages) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The user study (Section 7.4.5-7.4.6) uses Mann-Whitney U test with permutation testing (10,000 resamples) and reports p-values for each task comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4.6 reports effect sizes (r) derived from the standardized U statistic for each task comparison, e.g., r=0.788 for Task 2 EG1 vs CG1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The user study uses 32 participants (8 per group) with no power analysis or justification for the sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for the benchmark experiments (Tables 5-6). Individual user times are shown in Table 7 but no aggregate variance metrics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against zero-shot, few-shot, hand-crafted prompt, and DSPy baselines for order recovery (Table 3), and evaluates Cursor, Claude Code, and CoEdPilot with/without EditFlow (Tables 5-6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Cursor CLI (2025.09.18), Claude Code (1.0.113), CoEdPilot (2024), and DSPy — all contemporary systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison of Original vs w/ EditFlow for each baseline system (Tables 5-6) serves as an ablation showing the contribution of the flow-aware optimization component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metric categories are used: flow categories (Keep/Jump/Revert/Break), precision/recall/F0.5, and resource usage metrics (latency, tokens, cost)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 7.4) presents a user study with 32 developers evaluating real-world task completion and perceived recommendation quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1.2: 'we split the dataset at the commit level in a 7:3 ratio, ensuring that all samples from the same commit are assigned to the same split' — preventing intra-commit data leakage."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per baseline system (Cursor, Claude Code, CoEdPilot), per task in the user study (Tasks 1-3), and per flow category (Keep/Jump/Revert/Break)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 provides a detailed failure analysis with two specific failure modes (false rejection due to k-context sensitivity and acceptance of incorrect flow-keeping edits), including concrete examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that EditFlow does not show statistically significant improvement on Tasks 1 and 3 in the user study, and discusses why (Section 7.4.6). Recall decreases by 7.09% on average."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (63.81% order reconstruction improvement, 75% flow violation reduction, 66.99% precision improvement, 25.11% faster completion) are all supported by Tables 3-7 in the results sections."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'EditFlow improves precision' are supported by controlled comparisons (Original vs w/ EditFlow on same benchmarks). The user study uses random group assignment with statistical testing."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to benchmark 'Code Edit Recommendation Systems' generally, but evaluation is limited to Python commits only. The paper acknowledges this in Section 9 (External validity) but the title and abstract do not bound to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 (Threats to Validity) discusses multiple alternative explanations: digital twin assumes correct developer decisions, edit-order data may not be optimal, violation-based metric may introduce optimistic bias, and LLM stochasticity."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames 'mental flow' as a cognitive construct and operationalizes it via pairwise edit order relations. It discusses the gap between the proxy (edit ordering) and the construct (cognitive flow state), noting that 'our operationalization through the Keep/Jump/Revert/Break taxonomy may only approximate developers' cognitive states' (Section 9)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1.1 specifies 'Claude-Sonnet-4-20250514' with exact snapshot date. Section 7.3.3 gives exact versions: 'Claude Code (Version 1.0.113), Cursor CLI (Version 2025.09.18-7ae6800)'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The auto-tuned prompt is central to the method but is not included in the paper. The paper says 'For the detailed learned prompt, please refer to our anonymous website [3]' (Section 6.1). The prompt itself is not in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1.1: 'maximum output length of 4096 tokens and a temperature of 0.7. The auto-tuning underwent 5 epochs and a batch size of 32.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates Cursor CLI and Claude Code as black boxes ('relied on their default underlying models without manually specifying a particular model'). While EditFlow's own wrapper is described, the underlying systems' scaffolding is not. Per schema rules, this should be NA for the third-party tools, but EditFlow's own scaffolding (filter and re-rank) is described. However, the digital twin integration approach is only briefly described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3.2 documents commit selection criteria: '(1) containing 5-10 edit hunks across at least 2 source files; (2) involving real user authorship; (3) excluding merge commits and filename changes; (4) maintaining ASCII-only content with meaningful code modifications.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Threats to Validity' provides a substantive discussion of external, construct, and internal validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 discusses specific threats: Python-only benchmark composition, the single-trajectory limitation of edit order data, the digital twin's assumption of correct developer decisions, and LLM stochasticity affecting prediction stability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 explicitly states limitations: 'our data composition may limit the generalizability of our findings to other programming languages, development workflows, or industrial settings.' Also acknowledges the digital twin's simplifying assumptions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The annotated dataset and experiment results are stated as available at the anonymous website. The industrial dataset cannot be released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1.2 describes the annotated dataset: '100 commits from the 45 most-starred open-source GitHub Python repositories, comprising 772 edit hunks and 1,747 directed edges.' Section 7.2.2 describes the industrial dataset: '500 commits from Jun. 2025 to Aug. 2025, containing 3,059 edit hunks.'"
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4.2: 'We recruited 32 participants from 2 universities' with demographics (age 20-30, CS students from undergraduate to PhD, 4.5 days/week coding, 90% prior AI tool experience). Footnote 3 describes employee consent and anonymization for industrial data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The annotation pipeline is described in Section 6.1: independent annotation by two authors, consensus resolution, 20 minutes per commit, 77 person-hours total. The data split is documented (7:3 at commit level)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists multiple funding sources: National Natural Science Foundation of China, Ministry of Education Singapore, National Research Foundation Singapore, AI Singapore Programme, and Cyber Security Agency of Singapore."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Shanghai Jiao Tong University, National University of Singapore, and ByteDance. Bo Jiang from ByteDance is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are government research agencies (NSFC, Singapore MOE, NRF) that have no commercial stake in the outcome. ByteDance affiliation exists but they are not listed as a funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. One author is from ByteDance, which develops code editing tools, but no financial interest disclosure is included."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Claude-Sonnet-4-20250514 and Cursor/Claude Code for evaluation but does not state the training data cutoff dates for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the GitHub commits used in benchmarks could have appeared in the training data of the LLMs used for order recovery or evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses commits from popular GitHub repositories (most-starred Python repos). These are highly likely to be in training data for Claude and other models, but contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration mentioned for the user study with 32 participants."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the user study. Footnote 3 mentions employee consent for industrial data but not ethics approval for the user study."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4.2: participants aged 20-30, CS students (undergrad to PhD), coding 4.5 days/week on average, 90% with prior AI tool experience."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4.2: 'All participants are required to complete a pre-study questionnaire to collect their background information, including educational level, programming proficiency, and prior experience with AI-assisted programming tools.'"
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes four groups (CG1, EG1, CG2, EG2) but does not explain how participants were assigned to groups (randomization procedure not described)."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether participants knew which condition they were in (with or without EditFlow). Blinding is not discussed."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 shows results for all 32 participants (P1-P32), 8 per group, with no apparent dropouts. All participants completed all 3 tasks."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 5-6 report per-query resource usage including latency (seconds), token usage (K), and monetary cost ($) for each system with and without EditFlow."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, total API spend, total experiment cost) is reported. Only per-query costs are shown."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of random seed sensitivity. Section 9 acknowledges LLM stochasticity but does not report results across multiple seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many times each experiment was run. The digital twin simulation appears to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The prompt auto-tuning uses 5 epochs but the total search budget (number of candidate prompts evaluated, compute cost) is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 describes the prompt selection procedure: accuracy on the full training set is used to select the best prompt at each epoch, which is a clearly defined selection criterion."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The user study performs multiple statistical comparisons (3 tasks × 2 system pairs = 6 tests) but no multiple comparison correction (Bonferroni, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own EditFlow system against baselines without acknowledging author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Tables 5-6 report resource usage (latency, tokens, cost) alongside performance for each system, allowing compute-performance comparison. The paper discusses the additional overhead of EditFlow (1.71s latency, 6.58K tokens, $0.03 per query)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 (Construct validity) explicitly discusses: 'The notion of mental-flow alignment is inherently abstract, and our operationalization through the Keep/Jump/Revert/Break taxonomy may only approximate developers' cognitive states.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates Cursor, Claude Code, and CoEdPilot as bundled products/tools. The scaffold IS the thing being tested, so this criterion does not apply."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark commits existed before the LLM training cutoffs. Top-starred GitHub repos are very likely in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The digital twin provides commit messages as edit descriptions to the SUTs, which could leak information about the expected edits. This is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1.2: 'To avoid intra-commit data leakage, we split the dataset at the commit level in a 7:3 ratio, ensuring that all samples from the same commit are assigned to the same split.'"
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used beyond the commit-level split. No canary strings, membership inference, or decontamination."
       }
     }
   },
   "claims": [
     {
       "claim": "68.81% of model recommendations disrupt developers' ongoing mental flow, including 8.83% that are technically correct but ill-timed.",
       "evidence": "Table 1 (Section 5): Cursor shows 28.23% Keep, Claude Code 34.16% Keep, with remaining edits in Jump/Revert/Break categories. Analysis of 50 real-world commits.",
       "supported": "moderate"
     },
     {
       "claim": "Auto-tuned prompt achieves 87.26% accuracy on edit order recovery, a 63.81% relative improvement over the best baseline (53.39%).",
       "evidence": "Table 3 (Section 7.1.5): Accuracy comparison across zero-shot (50.63%), few-shot (47.42%), hand-crafted (53.27%), DSPy (53.39%), and auto-tuned (87.26%).",
       "supported": "strong"
     },
     {
       "claim": "EditFlow reduces flow violations by over 75% on real-world industrial data.",
       "evidence": "Table 4 (Section 7.2.4): Auto-tuned prompt achieves only 30 violations vs. 121 for best baseline (hand-crafted prompt).",
       "supported": "strong"
     },
     {
       "claim": "EditFlow improves recommendation precision by an average of 66.99% across systems and benchmarks.",
       "evidence": "Tables 5-6 (Section 7.3.5): Precision improvements shown for Cursor, Claude Code, and CoEdPilot on both large-scale and human-annotated benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "User study shows 25.11% faster task completion with EditFlow.",
       "evidence": "Table 7 (Section 7.4.6): Average completion times compared across experiment and control groups, but statistical significance only achieved for Task 2 (p=0.0004). Tasks 1 and 3 not statistically significant.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "rct"],
   "key_findings": "EditFlow demonstrates that existing AI code editing tools (Cursor, Claude Code, CoEdPilot) disrupt developers' mental flow in 65-72% of recommendations. A prompt auto-tuning approach recovers edit ordering with 87.26% accuracy. Applied as a post-processing wrapper, EditFlow improves recommendation precision by 66.99% on average and reduces flow violations by 75%. A user study with 32 participants shows 25.11% faster task completion, with statistically significant gains on the most challenging task but not on easier tasks.",
   "red_flags": [
     {
       "flag": "Small user study with no randomization description",
       "detail": "32 participants split into 4 groups of 8 each. No power analysis, no description of how participants were assigned to groups, no blinding. Statistical significance achieved only on 1 of 3 tasks for both system comparisons."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "Benchmarks use commits from the most-starred GitHub Python repositories, which are very likely in the training data of the LLMs used. No contamination analysis is performed."
     },
     {
       "flag": "Self-fulfilling evaluation concern partially addressed",
       "detail": "The paper acknowledges that flow-aware metrics could be self-fulfilling (Section 7.3.2) and includes a human-annotated benchmark to mitigate, but the large-scale benchmark still uses LLM-inferred partial orders as ground truth."
     },
     {
       "flag": "ByteDance affiliation with no conflict disclosure",
       "detail": "One author is from ByteDance, which develops code editing tools. No competing interests statement is provided."
     },
     {
       "flag": "Selective reporting of the 25.11% claim",
       "detail": "The headline '25.11% faster task completion' is an average across tasks and groups, but only Task 2 shows statistically significant improvement. Tasks 1 and 3 show no significant difference. The abstract presents this as an unqualified finding."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "doi": "10.48550/arXiv.2507.09089",
       "relevance": "Key RCT finding that developers completed tasks 19% slower with AI assistance, directly motivating EditFlow's premise."
     },
     {
       "title": "CoEdPilot: Recommending Code Edits with Learned Prior Edit Relevance, Project-wise Awareness, and Interactive Nature",
       "authors": ["Chenyan Liu", "Yufan Cai", "Yun Lin"],
       "year": 2024,
       "doi": "10.1145/3650212.3652142",
       "relevance": "Key baseline system for code edit recommendation that EditFlow wraps and optimizes."
     },
     {
       "title": "Codeplan: Repository-level coding using llms and planning",
       "authors": ["Ramakrishna Bairi"],
       "year": 2024,
       "doi": "10.1145/3643757",
       "relevance": "Integrates LLMs with static analysis for code change reasoning, relevant to repository-level code editing."
     },
     {
       "title": "The SPACE of Developer Productivity: There's more to it than you think",
       "authors": ["Nicole Forsgren", "Margaret-Anne Storey", "Chandra Maddila", "Thomas Zimmermann"],
       "year": 2021,
       "doi": "10.1145/3453928",
       "relevance": "Industry-standard framework for developer productivity that positions flow as a core dimension."
     },
     {
       "title": "DevEX: What actually drives productivity?",
       "authors": ["Abi Noda", "Margaret-Anne Storey", "Nicole Forsgren", "Michaela Greiler"],
       "year": 2023,
       "doi": "10.1145/3610285",
       "relevance": "Developer experience framework positioning flow state as a first-class driver of effective software development."
     },
     {
       "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
       "authors": ["Omar Khattab"],
       "year": 2024,
       "relevance": "Baseline prompt optimization framework compared against EditFlow's auto-tuning approach."
     },
     {
       "title": "The impact of LLM-assistants on software developer productivity: A systematic literature review",
       "authors": ["Amr Mohamed", "Maram Assi", "Mariam Guizani"],
       "year": 2025,
       "doi": "10.48550/arXiv.2507.03156",
       "relevance": "SLR documenting both positive effects and downsides of LLM coding assistants on developer productivity."
     },
     {
       "title": "Good Vibrations? A Qualitative Study of Co-Creation, Communication, Flow, and Trust in Vibe Coding",
       "authors": ["Veronica Pimenova", "Sarah Fakhoury", "Christian Bird", "Margaret-Anne Storey", "Madeline Endres"],
       "year": 2025,
       "doi": "10.48550/arXiv.2509.12491",
       "relevance": "Qualitative evidence on developer flow disruption during AI-assisted coding (vibe coding)."
     },
     {
       "title": "It's weird that it knows what i want: Usability and interactions with copilot for novice programmers",
       "authors": ["James Prather"],
       "year": 2023,
       "doi": "10.1145/3617367",
       "relevance": "Documents cognitive load and flow disruption from AI coding assistants for novice programmers."
     },
     {
       "title": "Overwatch: Learning Patterns in Code Edit Sequences",
       "authors": ["Yuhao Zhang"],
       "year": 2022,
       "doi": "10.1145/3563302",
       "relevance": "Prior work on learning code edit patterns from IDE-logged traces, directly related to edit recommendation."
     }
   ]
 }