ai-research-survey

scan.json (24545B)

---

 {
   "paper": {
     "title": "In-IDE Human-AI Experience in the Era of Large Language Models; A Literature Review",
     "authors": [
       "Agnia Sergeyuk",
       "Sergey Titov",
       "Maliheh Izadi"
     ],
     "year": 2024,
     "venue": "ICSE 2024",
     "arxiv_id": "2401.10739",
     "doi": "10.1145/3643796.3648463"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This literature review of 36 papers (2020–2024) identifies three research branches in in-IDE Human-AI Experience: Design of Interaction (14 papers on UI principles), Impact of Interaction (13 papers on workflow and productivity effects), and Quality of Interaction (9 papers on correctness, comprehensibility, and security). The authors find that AI tools increase productivity but may trade off code quality, that security risks vary by model (up to 40% vulnerability rate for C code), and that over-reliance on AI remains a challenge especially for novice developers. The paper proposes three future research directions: task-specific UI design, trust-building mechanisms, and code readability as an alignment criterion.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No analysis code or scripts are released. The paper provides a dataset on Zenodo but no code to reproduce the analysis or categorization."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The authors provide their curated dataset on Zenodo (https://doi.org/10.5281/zenodo.10290921), referenced in Section 4.2 and footnote 2."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. As a qualitative survey, no computational environment was needed, but no tooling for data extraction or analysis is specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the search string and databases are listed, there are no instructions for how to replicate the filtering and categorization process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative literature review with no statistical experiments or quantitative aggregation."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative statistical claims are made; the paper categorizes and summarizes literature qualitatively."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; effect sizes are not applicable to this qualitative literature survey."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey sample size is determined by the search strategy, not experimental power analysis. Not applicable for this paper type."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted; variance reporting is not applicable to a qualitative literature review."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No prior surveys are compared against. The authors claim 'the absence of a literature survey in this domain' but do not systematically compare with related reviews or surveys in adjacent fields."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baselines are included, so their contemporariness cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate; this is a literature survey."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted that would require evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate; this is a survey paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with train/test splits are conducted."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by the three identified categories: Design (14 papers), Impact (13 papers), and Quality (9 papers), each discussed in dedicated subsections of Section 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses challenges and failure modes identified in reviewed work: over-reliance on AI, security vulnerabilities reaching 40% for C programs, code quality trade-offs, and compatibility issues with AI tools."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings from the reviewed literature including productivity-quality trade-offs, security vulnerabilities, over-reliance concerns for novices, and limitations of AI assistants for non-trivial tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims three research branches were identified from 36 papers, which matches the analysis in Section 3. The proposed future directions (task-specific UI, trust, readability) are discussed in Section 4.1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper relays causal claims from reviewed work as conclusions (e.g., 'Using AI tools increases productivity' in Section 3, 'in-IDE Human-AI Interaction significantly affects and changes the developers' workflow') without assessing whether the underlying study designs actually support causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states 'it is evident that the field has three branches' as if this is the definitive structure of the field, generalizing from only 36 papers (22 from ArXiv). The title claims to cover 'the Era of Large Language Models' broadly. While temporal bias is acknowledged, the generalizations from 36 papers to the entire field are not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (4.2) discusses methodological limitations (sampling bias, temporal bias, source reliability) but does not consider alternative categorizations of the field or alternative interpretations of the patterns found."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper analyzes 36 papers but frames findings as describing 'the field' of in-IDE HAX. The gap between a 36-paper sample (61% non-peer-reviewed ArXiv papers) and claims about the entire field's structure is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used in the paper's own methodology; this is a manual literature review."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used; this is a manual literature review."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No AI models or computational experiments are used in the paper's methodology."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; this is a manual literature survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The search string and inclusion/exclusion criteria are stated (Section 2), but the filtering from 211 to 36 papers is presented as a single step with no intermediate stage counts, no screening agreement rates, and no documentation of the categorization process."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 'Threats to Validity' provides a dedicated discussion of potential validity concerns across four dimensions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses study-specific concerns: sampling bias from the chosen databases, temporal bias from the 2020–2024 timeframe, reliability concerns about ArXiv preprints, and interpretation bias from categorizing the large corpus."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT cover. Limitations are framed as potential biases rather than clear scope exclusions. No specific statements about excluded topics, populations, or settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo dataset (DOI: 10.5281/zenodo.10290921) is provided, containing the full set of extracted data from the 36 reviewed papers."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the data collection: four databases (ACM, DBLP, IEEE, ArXiv), the full search string, inclusion/exclusion criteria, and the resulting 211→36 paper counts."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper search strategy is documented: databases selected, search string provided, and filtering criteria described. Table 1 breaks down source venues."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline jumps from 211 initial papers to 36 final papers with no intermediate stage counts. The categorization of papers into Design/Impact/Quality is not documented as a process — no inter-rater agreement, no coding procedure described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper, despite two of three authors being affiliated with JetBrains Research, a commercial entity with products in the reviewed space."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: JetBrains Research (Sergeyuk, Titov) and Delft University of Technology (Izadi). However, the paper does not acknowledge the conflict that JetBrains produces IDEs and an AI assistant plugin."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two authors are employed by JetBrains Research, and JetBrains has a direct commercial interest in positive portrayals of in-IDE AI tools (JetBrains produces IntelliJ IDEA with an AI Assistant plugin, mentioned in footnote 1). The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. JetBrains' commercial interest in the reviewed product space is not acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature survey that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature survey that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature survey that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational method to cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a search string and databases but lacks a PRISMA flow diagram, protocol registration, intermediate-stage filtering counts, or screening agreement rates. The jump from 211 to 36 papers is presented as one step with no staged breakdown."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "No quality scoring, risk-of-bias assessment, or structured evaluation of the 36 included papers is performed. ArXiv preprints (22/36, 61%) are treated identically to peer-reviewed publications. The survey launders varying source quality without assessment."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that ArXiv 'encourages publishing negative or null results' (Section 4.2) but does not systematically address publication bias, funnel plots, or whether the positive-result skew in the reviewed papers affects conclusions."
       }
     }
   },
   "claims": [
     {
       "claim": "In-IDE HAX research divides into three primary research branches: Design, Impact, and Quality of Interaction.",
       "evidence": "Categorization of 36 papers: 14 on Design, 13 on Impact, 9 on Quality (Section 3). Each category is discussed in a dedicated subsection.",
       "supported": "moderate"
     },
     {
       "claim": "Using AI tools increases productivity but may involve a trade-off in code quality.",
       "evidence": "Cited from reviewed studies [10, 13, 17, 35, 36, 39, 43] in Section 3 (Impact). No quality assessment of these studies or quantified meta-analysis is provided.",
       "supported": "weak"
     },
     {
       "claim": "The user interface of in-IDE AI assistance affects the usefulness of the tool and should be built thoughtfully.",
       "evidence": "Synthesized from Design category studies [16, 20, 34, 38] in Section 3. Design principles are listed but no quantified evidence is presented.",
       "supported": "weak"
     },
     {
       "claim": "Security risks depend on the model used, with vulnerability rates reaching 40% for generated C programs.",
       "evidence": "Cited from reviewed studies [13, 25, 30] in Section 3 (Quality). The 40% figure is attributed to specific C-language evaluation in the reviewed work.",
       "supported": "moderate"
     },
     {
       "claim": "AI assistance for novices positively influences programming education but introduces challenges like over-reliance.",
       "evidence": "Cited from studies [2, 15, 26] in Section 3 (Impact). No quality assessment or meta-analysis of these findings is conducted.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Two of three authors are from JetBrains Research. JetBrains produces IntelliJ IDEA with an AI Assistant plugin (footnote 1 explicitly references it). The paper reviews in-IDE AI research without disclosing this commercial conflict or discussing how it might influence the review's framing."
     },
     {
       "flag": "Heavy reliance on non-peer-reviewed sources without quality assessment",
       "detail": "22 of 36 papers (61%) are ArXiv preprints without peer review. No quality assessment, risk-of-bias scoring, or distinction is made between peer-reviewed and non-peer-reviewed sources. This risks laundering weak or unvalidated findings as established knowledge."
     },
     {
       "flag": "Opaque filtering pipeline",
       "detail": "211 papers were reduced to 36 in a single described step with no intermediate counts, no screening agreement rates, and no documentation of how categorization decisions were made. The subjectivity of the 'thematically irrelevant' exclusion criterion is not addressed."
     },
     {
       "flag": "No structured quality assessment of reviewed papers",
       "detail": "All 36 papers are treated as equally valid evidence regardless of their methodology, sample size, or rigor. The survey synthesizes conclusions from studies of varying quality without weighting or assessment, potentially amplifying weak findings."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2021,
       "arxiv_id": "2108.09293",
       "relevance": "Security evaluation of LLM code generation — directly relevant to AI code assistant safety."
     },
     {
       "title": "Grounded Copilot: How Programmers Interact with Code-Generating Models",
       "authors": ["Shraddha Barke", "Michael B. James", "Nadia Polikarpova"],
       "year": 2023,
       "relevance": "Empirical study of programmer-AI interaction patterns with code generation, core to understanding LLM programming workflows."
     },
     {
       "title": "GitHub Copilot AI pair programmer: Asset or Liability?",
       "authors": ["Arghavan Moradi Dakhel", "Vahid Majdinasab", "Amin Nikanjam", "Foutse Khomh", "Michel C. Desmarais"],
       "year": 2023,
       "arxiv_id": "2206.15331",
       "relevance": "Evaluation of Copilot code correctness and comprehensibility — directly assesses AI code generation quality."
     },
     {
       "title": "Expectation vs. Experience: Evaluating the Usability of Code Generation Tools Powered by Large Language Models",
       "authors": ["Priyan Vaithilingam", "Tianyi Zhang", "Elena L. Glassman"],
       "year": 2022,
       "relevance": "Usability evaluation of LLM-powered code generation tools — key evidence on developer experience with AI assistants."
     },
     {
       "title": "A Large-Scale Survey on the Usability of AI Programming Assistants: Successes and Challenges",
       "authors": ["Jenny T. Liang", "Chenyang Yang", "Brad A. Myers"],
       "year": 2023,
       "arxiv_id": "2303.17125",
       "relevance": "Large-scale survey of AI programming assistant usability, directly relevant to understanding developer productivity claims."
     },
     {
       "title": "Productivity Assessment of Neural Code Completion",
       "authors": ["Albert Ziegler", "Eirini Kalliamvakou", "X. Alice Li", "Andrew Rice"],
       "year": 2022,
       "relevance": "Productivity measurement of neural code completion — key evidence for AI coding assistant impact claims."
     },
     {
       "title": "An Empirical Evaluation of GitHub Copilot's Code Suggestions",
       "authors": ["Nhan Nguyen", "Sarah Nadi"],
       "year": 2022,
       "relevance": "Empirical evaluation of Copilot suggestion quality — benchmark for AI code generation correctness."
     },
     {
       "title": "Is GitHub's Copilot as Bad as Humans at Introducing Vulnerabilities in Code?",
       "authors": ["Owura Asare", "Meiyappan Nagappan", "N. Asokan"],
       "year": 2023,
       "arxiv_id": "2204.04741",
       "relevance": "Comparative security assessment of AI-generated vs human-written code vulnerabilities."
     },
     {
       "title": "Lost at C: A User Study on the Security Implications of Large Language Model Code Assistants",
       "authors": ["Gustavo Sandoval", "Hammond Pearce", "Teo Nys", "Ramesh Karri", "Siddharth Garg", "Brendan Dolan-Gavitt"],
       "year": 2023,
       "arxiv_id": "2208.09727",
       "relevance": "User study on security implications of LLM code assistants — combines human factors with security evaluation."
     },
     {
       "title": "Reading Between the Lines: Modeling User Behavior and Costs in AI-Assisted Programming",
       "authors": ["Hussein Mozannar", "Gagan Bansal", "Adam Fourney", "Eric Horvitz"],
       "year": 2023,
       "arxiv_id": "2210.14306",
       "relevance": "Models user behavior and costs during AI-assisted programming — evidence for workflow transformation claims."
     },
     {
       "title": "Is GitHub Copilot a Substitute for Human Pair-Programming? An Empirical Study",
       "authors": ["Saki Imai"],
       "year": 2022,
       "relevance": "Empirical comparison of AI assistance vs human pair programming — directly tests AI productivity claims."
     },
     {
       "title": "Guidelines for Human-AI Interaction",
       "authors": ["Saleema Amershi", "Dan Weld", "Mihaela Vorvoreanu", "Adam Fourney"],
       "year": 2019,
       "relevance": "Foundational HAX design guidelines that inform the paper's framework for evaluating in-IDE AI interactions."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Provides a categorized overview of the field and future directions, useful for researchers orienting in the space, but nothing directly usable by practitioners."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms expected findings about AI tools increasing productivity with quality trade-offs; no contrarian insights."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Mentions security vulnerabilities in AI-generated code (up to 40% for C) and over-reliance risks, but these are synthesized from existing work, not novel findings."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenges to established positions, straightforward literature categorization."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo. The Zenodo dataset is the only artifact."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "JetBrains (IntelliJ makers) is a recognizable brand; the paper discusses GitHub Copilot prominently. Published at ICSE."
     }
   }
 }