ai-research-survey

scan.json (26704B)

---

 {
   "paper": {
     "title": "A Multi-agent Onboarding Assistant based on Large Language Models, Retrieval Augmented Generation, and Chain-of-Thought",
     "authors": [
       "Andrei-Cristian Ionescu",
       "Sergey Titov",
       "Maliheh Izadi"
     ],
     "year": 2025,
     "venue": "FSE'25 (SIGSOFT FSE Companion)",
     "arxiv_id": "2503.23421",
     "doi": "10.1145/3696630.3728611"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our tool, source code [7], and demonstration video [6] are publicly available.' Reference [7] points to https://onboarding.software/register?colab=true."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The study was conducted on a closed-source project (Section 5: 'We chose a closed-source project to avoid data contamination'). Neither the study telemetry data, questionnaire responses, nor the evaluated codebase are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or environment setup details are provided in the paper. Only the embedding model (OpenAI text-embedding-3-large) and FAISS are named without version numbers."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided for either the system setup or the user study. The paper describes the architecture but not how to replicate the evaluation."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Only means and standard deviations are reported (M=3.26, SD=0.86 for helpfulness; M=3.0, SD=0.96 for ease). No confidence intervals or error bars are provided, which would be especially important given n=8."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. The paper reports only descriptive statistics (mean and SD) without any inferential tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. Only raw means and standard deviations are provided with no baseline for comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of n=8 is not justified. Section 5 acknowledges 'Our pilot study involved eight participants on one codebase due to the limitations of the demonstration project' but provides no power analysis or formal justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported for both metrics: helpfulness (SD=0.86) and ease of onboarding (SD=0.96). These represent variance across the eight participants."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No formal baseline comparison is included. The paper mentions GitHub Copilot and ChatGPT informally but does not compare against them. Section 6 notes participants 'who used mainstream AI coding tools in their daily workflow noted... Onboarding Buddy gave more context-specific advice' but this is anecdotal, not a formal comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all. The paper acknowledges the need for 'rigorous A/B testing' in future work (Section 6) but does not perform any baseline comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (Contextualization Agent, Onboarding Agent, Step Processor, Message Enhancer, RAG, CoT, planning scratchpad) but no ablation study is performed to measure individual component contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation dimensions are reported: perceived helpfulness (M=3.26), ease of onboarding (M=3.0), and task completion rate (7/8 participants completed all tasks)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation is a human user study with 8 participants performing onboarding tasks and providing ratings on helpfulness and ease of onboarding (Section 5-6)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a user study evaluating a tool, not a machine learning evaluation with train/test splits. The concept of a held-out test set is structurally inapplicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Three distinct tasks are used (Setup Task, New Payment Option Task, Questionnaire Duplication Task) but results are only reported as aggregate means across all tasks and participants. No per-task breakdown of ratings is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 briefly mentions 'a number of users experienced minor hiccups' on setup and questionnaire duplication tasks, and one participant had coding errors. But there is no systematic error analysis, no qualitative examples of failures, and no discussion of where the approach breaks down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports user feedback requesting improvements: more context awareness, better UI, faster response times. Section 7 acknowledges 'identified shortcomings' and the need to enhance 'technical reliability.' These constitute negative findings from the evaluation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The reported statistics (M=3.26, SD=0.86; M=3.0, SD=0.96) match the results section. However, the abstract claims 'Onboarding Buddy holds great potential for enhancing developer productivity and satisfaction' which is not supported by an n=8 pilot study with no baseline comparison. The claim that it 'uniquely integrates' CoT with RAG 'tailored specifically for dynamic onboarding contexts' is a design claim without comparative evidence."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies the system causes improved onboarding (Section 1: 'improve onboarding', Section 7: 'easy-to-use and useful tool that helps users'). However, there is no control group, no randomization, and no baseline — the study design cannot support causal claims about the system's effect on onboarding outcomes."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 acknowledges 'eight participants on one codebase... This may affect generalizability.' However, the title ('A Multi-agent Onboarding Assistant') and conclusion ('the Onboarding Buddy is an easy-to-use and useful tool that helps users in performing development tasks') make unbounded claims. The abstract states 'great potential for enhancing developer productivity and satisfaction' without qualification."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. Novelty effect, Hawthorne effect (being observed), selection bias in participants, or the possibility that any IDE-integrated assistant would achieve similar ratings are not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures 'perceived helpfulness' and 'ease of onboarding' on self-report Likert scales but frames results as evidence of 'enhancing developer productivity and satisfaction' (abstract). The gap between subjective ratings (proxy) and actual productivity improvement (claimed outcome) is never acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'large language models' throughout but never specifies which LLM powers the agents. Only the embedding model is named (OpenAI text-embedding-3-large, Section 4.2) without a version date. The core LLM is completely unspecified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided anywhere in the paper. The agents' behaviors are described in natural language (e.g., 'Allows the user interaction to be conversational') but the actual system prompts, instructions, or templates used are not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported. Only RAG parameters are mentioned: chunk size (2000 characters), overlap (200 characters), similarity threshold (0.1), top-k (5 documents)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent architecture is described in detail across Sections 3-4 with Figures 1-2. Four agents are specified (Contextualization, Onboarding, Step Processor, Message Enhancer), along with the planning scratchpad mechanism, retrieval tools, memory storage, and the CoT generation workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The RAG chunking pipeline is described (2000 char chunks, 200 overlap, FAISS indexing) but data preprocessing for the evaluation is not documented. How telemetry was collected, how questionnaire responses were aggregated from per-task to overall means, and what data cleaning was applied are not described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. A single sentence in Section 5 notes 'This may affect generalizability' and the conclusion mentions 'identified shortcomings' without substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The only specific threat mentioned is the small sample size and single codebase (Section 5: 'eight participants on one codebase'). No discussion of construct validity, internal validity threats (no control group, novelty effect), or external validity beyond the generic generalizability mention."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While acknowledging the small sample, it does not state specific things it did not test or claim. The conclusion makes broad unbounded claims ('easy-to-use and useful tool that helps users in performing development tasks')."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the raw questionnaire responses, telemetry data, nor per-participant ratings are made available. Only aggregate statistics (means and SDs) are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes the data collection: three onboarding tasks, project telemetry to track progress, post-task questionnaire with helpfulness and ease ratings on a 0-4 scale, and open-ended feedback. The tasks themselves are described in detail."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We gathered a sample of eight programmers' (Section 5) but provides no information about how they were recruited, from what population, or whether they were students, colleagues, or external participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how raw data was processed into the reported statistics. How telemetry was analyzed, how per-task ratings were aggregated to overall means, and whether any responses were excluded are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 acknowledges: 'This work was conducted as part of the AI for Software Engineering (AI4SE) collaboration between JetBrains and Delft University of Technology. The authors gratefully acknowledge the financial support provided by JetBrains.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Sergey Titov is from JetBrains Research. The tool is an IntelliJ Plugin (JetBrains product ecosystem). The JetBrains affiliation is transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "JetBrains funds the research and has a direct commercial interest in demonstrating that AI-powered tools enhance their IDE ecosystem. The Onboarding Buddy is an IntelliJ plugin, making JetBrains a non-independent funder with a stake in positive outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. While the JetBrains funding and affiliation are disclosed, there is no formal declaration of financial interests, patents, or equity."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates a tool through a user study, not a pre-trained model's capability on a benchmark. The contamination category is structurally inapplicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model knowledge is performed. The study evaluates a tool through human participants on a closed-source project."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The paper evaluates a tool via user study, not model performance on a benchmark dataset."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, or any registry). The study design was not committed to before data collection."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval despite conducting a study with human participants performing tasks and providing personal evaluations."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Participants are described only as 'eight programmers' (Section 5). No demographics are reported — no experience level, programming background, years of experience, or familiarity with LLM tools."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated. The paper says 'We gathered a sample of eight programmers' with no description of who was eligible or how they were selected."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a single-condition study where all participants use the same tool. There are no experimental conditions requiring randomized assignment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Single-condition study with no comparison group. Blinding is not feasible when all participants use the same tool and there is nothing to blind against."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports that all 8 participants completed the study: 'Seven out of eight participants successfully completed all tasks, with only one participant encountering minor coding errors on a single task.' No dropout occurred."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs, token consumption, or response latency is reported. Users mentioned wanting 'faster response times' (Section 6) but no actual latency measurements are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, API spend, or hardware requirements are stated for running the system."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [],
   "claims": [
     {
       "claim": "Onboarding Buddy achieves a mean perceived helpfulness rating of M=3.26 (SD=0.86) out of 4 across all participants and tasks.",
       "evidence": "Section 6 reports aggregated questionnaire results from 8 participants across 3 tasks on a 0-4 scale.",
       "supported": "moderate"
     },
     {
       "claim": "Ease of onboarding averaged M=3.0 (SD=0.96) out of 4.",
       "evidence": "Section 6 reports aggregated questionnaire results from 8 participants across 3 tasks on a 0-4 scale.",
       "supported": "moderate"
     },
     {
       "claim": "Seven out of eight participants successfully completed all tasks with close to 100% accuracy.",
       "evidence": "Section 6: 'Seven out of eight participants successfully completed all tasks, with only one participant encountering minor coding errors on a single task.' Verified via collected telemetry.",
       "supported": "strong"
     },
     {
       "claim": "Onboarding Buddy provides more context-specific advice than mainstream AI coding tools like GitHub Copilot.",
       "evidence": "Section 6: 'participants who used mainstream AI coding tools in their daily workflow noted in their feedback that Onboarding Buddy gave more context-specific advice, particularly on setup tasks.' This is anecdotal participant feedback, not a controlled comparison.",
       "supported": "weak"
     },
     {
       "claim": "Onboarding Buddy holds great potential for enhancing developer productivity and satisfaction.",
       "evidence": "Abstract and Section 7. Based solely on the n=8 pilot study results with no baseline comparison and no actual productivity measurement.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "case-study",
     "qualitative"
   ],
   "key_findings": "Onboarding Buddy, a multi-agent LLM system with RAG and chain-of-thought reasoning, was evaluated in a pilot study with 8 programmers performing 3 onboarding tasks on a closed-source project. Participants rated perceived helpfulness at M=3.26/4 and ease of onboarding at M=3.0/4, with 7/8 completing all tasks. Users requested improvements including more specific code snippets, better UI, and faster response times. The evaluation lacks baseline comparisons and formal statistical analysis.",
   "red_flags": [
     {
       "flag": "Tiny sample size with broad claims",
       "detail": "Only 8 participants evaluated the system, yet the paper claims 'great potential for enhancing developer productivity and satisfaction.' This sample is far too small for the generalizations made, and no power analysis or sample size justification is provided."
     },
     {
       "flag": "No baseline comparison",
       "detail": "The system is evaluated in isolation with no control condition (e.g., onboarding without the tool, or using GitHub Copilot/ChatGPT). Without a baseline, it is impossible to attribute the outcomes to the tool rather than to the tasks being straightforward, the documentation being adequate, or other factors."
     },
     {
       "flag": "Non-independent funder evaluating own ecosystem",
       "detail": "JetBrains funds the research, employs a co-author (Sergey Titov), and the tool is an IntelliJ plugin — directly benefiting the JetBrains product ecosystem. This conflict of interest is not explicitly acknowledged beyond the acknowledgments section."
     },
     {
       "flag": "Core LLM unspecified",
       "detail": "The paper never discloses which large language model powers the agents. Without knowing the model, the results cannot be interpreted in the context of model capabilities, nor can the system be reproduced."
     },
     {
       "flag": "Recruitment bias unknown",
       "detail": "How the 8 participants were recruited is not described. If they were colleagues, students of the authors, or JetBrains employees, this would introduce significant selection bias. No demographics or selection criteria are reported."
     },
     {
       "flag": "No IRB/ethics approval for human subjects",
       "detail": "A user study with 8 participants collecting performance data and subjective ratings was conducted without any mention of ethics review or IRB approval."
     }
   ],
   "cited_papers": [
     {
       "title": "Extending source code pre-trained language models to summarise decompiled binaries",
       "authors": ["Ali Al-Kaswan", "Toufique Ahmed", "Maliheh Izadi", "Anand Ashok Sawant", "Premkumar Devanbu", "Arie van Deursen"],
       "year": 2023,
       "relevance": "Evaluates pre-trained language models for code understanding tasks (binary summarization), relevant to LLM capabilities for code."
     },
     {
       "title": "Leveraging large language models for enhancing the understandability of generated unit tests",
       "authors": ["Amirhossein Deljouyi", "Roham Koohestani", "Maliheh Izadi", "Andy Zaidman"],
       "year": 2024,
       "arxiv_id": "2408.11710",
       "relevance": "Uses LLMs to improve code test understandability, directly relevant to LLM-assisted software engineering productivity."
     },
     {
       "title": "Language models for code completion: A practical evaluation",
       "authors": ["Maliheh Izadi", "Jonathan Katzy", "Tim Van Dam", "Marc Otten", "Razvan Mihai Popescu", "Arie Van Deursen"],
       "year": 2024,
       "relevance": "Practical evaluation of LLMs for code completion, relevant to benchmarking AI coding assistants."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus", "Fabio Petroni", "Vladimir Karpukhin"],
       "year": 2021,
       "arxiv_id": "2005.11401",
       "relevance": "Foundational RAG paper used as the basis for the retrieval-augmented approach in this and many other LLM-based tools."
     },
     {
       "title": "Experiences from Using Code Explanations Generated by Large Language Models in a Web Software Development E-Book",
       "authors": ["Stephen MacNeil", "Andrew Tran", "Arto Hellas", "Joanne Kim", "Sami Sarsa", "Paul Denny"],
       "year": 2023,
       "relevance": "Studies LLM-generated code explanations for learning, directly relevant to AI-assisted developer onboarding and education."
     },
     {
       "title": "The Design Space of in-IDE Human-AI Experience",
       "authors": ["Agnia Sergeyuk", "Ekaterina Koshchenko", "Ilya Zakharov", "Timofey Bryksin", "Maliheh Izadi"],
       "year": 2024,
       "arxiv_id": "2410.08676",
       "relevance": "Explores the design space for human-AI interaction within IDEs, relevant to how AI coding assistants should be integrated."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Brian Ichter", "Fei Xia", "Ed Chi", "Quoc Le", "Denny Zhou"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational chain-of-thought prompting paper that inspired the CoT approach used in the onboarding agent architecture."
     },
     {
       "title": "Automatic Chain of Thought Prompting in Large Language Models",
       "authors": ["Zhuosheng Zhang", "Aston Zhang", "Mu Li", "Alex Smola"],
       "year": 2022,
       "arxiv_id": "2210.03493",
       "relevance": "Automated CoT generation technique that influenced the automated chain-of-thought approach in the Onboarding Buddy system."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "An IDE plugin for developer onboarding has clear practical value, though it's tied to IntelliJ and requires specific setup with a project's codebase."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "The finding that an LLM-RAG system can help with onboarding tasks confirms expectations rather than challenging them."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns are raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; a straightforward tool paper with positive preliminary results."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Source code and a demonstration video are publicly available, and there is a web registration portal, though setting up the full system requires effort."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "JetBrains is well-known in developer tooling but not a major AI research brand; the paper comes from a university-industry collaboration."
     }
   }
 }