ai-research-survey

scan.json (23208B)

---

 {
   "paper": {
     "title": "Design and Evaluation of an Assisted Programming Interface for Behavior Trees in Robotics",
     "authors": [
       "Jonathan Styrud",
       "Matteo Iovino",
       "Rebecca Stower",
       "Mart Kartašev",
       "Mikael Norrlöf",
       "Mårten Björkman",
       "Christian Smith"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.09772"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": [
     "rct"
   ],
   "key_findings": "BETR-GUI, combining LLMs, planning, genetic programming, and Bayesian optimization with a drag-and-drop editor, enables users to perform significantly better at robot programming tasks than manual-only programming. Ablations show that the planner and LLM are the critical components (removing either eliminates the advantage over manual), while removing GP or BO alone does not significantly degrade performance. Humans using the full system significantly outperform the AI assistant running alone (91.1 vs 88.1 mean score).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/jstyrud/BETR-GUI, referenced in Section III and footnotes."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Experimental data uploaded to the OSF repository (https://osf.io/ax5gb/overview), referenced in Sections V-D and VI."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python/PyQt5 and Unity but does not provide a requirements.txt, Dockerfile, or detailed dependency list with versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The GitHub repo is referenced but no README instructions or reproduction guide is detailed in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported in Tables IV, VI, and VIII for all fixed effects (e.g., 'Trial Order: 95% CI [3.0, 10.8]')."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Linear mixed models with F-tests and t-tests are used throughout (Section VI). Post-hoc pairwise comparisons with Tukey adjustment for multiple comparisons are reported in Tables V and IX. Likelihood ratio test used for rankings (χ²(5) = 404.2, p < .001)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pseudo R² values reported for models (R² = 0.62 for task performance, R² = 0.21 for SUS, R² = 0.04 for FULL vs NO_HUMAN). Regression coefficients (b values) with standard errors provided in all tables, giving effect magnitudes in the original scale."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E: 'The number of participants needed was determined a priori using a simulated power analysis with α = .05, suggesting 60 participants are sufficient to reach 80% power, assuming small-medium effects.'"
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations reported for all GUI variant scores (Table III, e.g., 'FULL: 91.14 (9.07)'), SUS scores (Table VII), and participant demographics (Table II)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "MANUAL_ONLY variant serves as baseline, described as 'largely similar to existing GUIs like Groot in capability' (Section V-A). NO_HUMAN ablation also serves as a baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The MANUAL_ONLY baseline is compared to Groot (2025), a contemporary commercial BT editor. The AI components build on recent prior work (2024-2025 references)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive ablation with 6 variants: FULL, MANUAL_ONLY, NO_BO, NO_GP, NO_LLM, NO_PLANNER (Section V-A). Each ablation removes one component to measure its contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics used: task performance score (Section V-B), System Usability Scale (Section V-F), and preference rankings (Section VI-D)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire study IS a human evaluation — 60 participants used the system and provided SUS scores and rankings."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a user study, not a model evaluation on a dataset split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by GUI variant (Tables III, V, VII, IX), trial number (Figures 8, 9), and task type (tested as fixed effect, found non-significant)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses failure modes: users not using locking functionality (74/120), users not trusting AI suggestions, and user quotes describing negative experiences with the AI assistant."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results reported: NO_BO and NO_GP ablations did not significantly differ from FULL; NO_LLM and NO_PLANNER did not significantly outperform MANUAL_ONLY. Hypotheses 1 and 2 only 'partially supported.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that BETR-GUI 'enables users to perform better' and 'humans using the full variant perform better than the AI assistant running on its own' are both supported by the statistical results in Section VI (Tables V and VI)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('enables users to perform better') are justified by the RCT design with randomized assignment, counterbalanced conditions, and controlled experiments (Section V-B). Mixed-model analysis accounts for individual differences."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Section VIII acknowledges tasks are 'highly simplified compared to actual robot applications' but the title and abstract frame results broadly as 'Assisted Programming Interface for Behavior Trees in Robotics' without bounding to the specific simplified scenarios and participant population tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses multiple alternative explanations: users not understanding how to use AI features, planner solving most of the task leaving little for learning algorithms, short experiment time disadvantaging learning methods, and trust issues with the AI assistant."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The score function (Eq. 1-4) is a composite of goal fulfillment, tree complexity, and energy penalties with manually-set weights. The paper does not discuss whether this proxy score reflects actual task quality in real robot programming. The gap between this artificial score and real-world robot programming effectiveness is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section III states 'The LLM model used in all experiments was GPT-4' without specifying a version (e.g., gpt-4-0613) or snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper references a 'slightly updated prompt' from prior work [12] but does not provide the actual prompt text. The method is described in natural language without the prompts themselves."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other LLM API parameters are stated. GP and BO hyperparameters are referenced as being available in the GitHub repo but not stated in the paper. Score weight values are also deferred to the repository."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The AI assistant pipeline is described in detail in Section III-D and Figure 2: seed BT → planner expansion → LLM error resolution → GP/BO optimization loop. Node locking, simulation evaluation, and user interaction points are all documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Score normalization procedure described in Section VI-A. Experimental randomization and counterbalancing described in Section V-B. One participant excluded due to a bug and replaced (Section V-E)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII (Future Work) opens with 'The main limitation of this work is that the benchmark tasks, out of necessity, are highly simplified compared to actual robot applications' and continues with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: simplified tasks vs. real applications, short experiment time disadvantaging learning algorithms, users not understanding how to use node locking, and trust issues with the AI. Section VII discusses these specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII explicitly states that tasks are simplified, more time and training would be needed for realistic tasks, and that component importance depends on other components in the system."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Data uploaded to OSF repository (https://osf.io/ax5gb/overview). Full analyses and supplementary materials referenced as available there."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section V-F describes the full procedure: consent form, demographic questionnaire, 5-minute instruction video, 5-minute familiarization, three 15-minute timed trials, SUS after each variant, final ranking."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E: 'We recruited 60 participants via flyers, mailing lists, social media, and word of mouth.' Demographics, age range (20-62), gender (10F/50M), and expertise levels reported."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The GUI automatically recorded actions and scores with timestamps (Section V-F). Score normalization explained (Section VI-A). One participant excluded and replaced with reason stated (Section V-E). Statistical analysis pipeline described with model selection criteria (Section VI)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding disclosed: 'This project is supported by the Wallenberg AI, Autonomous Systems, and Software Program (WASP) funded by the Knut and Alice Wallenberg Foundation.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: ABB Robotics, KTH, ETH Zürich, ABB Corporate Research, Ericsson. Multiple authors have ABB affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Wallenberg Foundation is an independent research foundation, not a product vendor. While ABB authors are listed, the funder (WASP/Wallenberg) has no direct commercial interest in the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. Multiple authors are affiliated with ABB (a robot manufacturer), and the tool could have commercial implications, but no financial interest disclosure is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. GPT-4 is used as a component in the system, but the evaluation is of the human-AI system, not of GPT-4's knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not evaluating a pre-trained model on a benchmark — this is a user study of a human-AI collaborative system."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not evaluating a pre-trained model on a benchmark — this is a user study of a human-AI collaborative system."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D: 'We pre-registered our hypotheses and planned analyses (both confirmatory and exploratory) on the Open Science Framework (OSF)' with link https://osf.io/ax5gb/overview."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "Section V-F states 'This study followed the ethical guidelines for Sweden' but does not mention IRB or ethics board approval specifically."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E: ages 20-62 (M=29.7, SD=8.9), 10 female/50 male, primarily engineering/CS students or professional developers. Table II reports familiarity scores across four domains."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No explicit inclusion or exclusion criteria stated. Participants described as 'primarily university students of engineering or computer science or professional software developers' but no screening criteria are mentioned."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B: 'We randomized all the experiments in advance, ensuring that all ablations occurred the same number of times, and that the order in which tasks and variants occurred was also counterbalanced.' Randomization code available on GitHub."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No mention of blinding. Participants could see which GUI variant they were using (e.g., presence/absence of AI features was obvious). No discussion of whether this awareness affected behavior."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E: 'After the experiments, we excluded one participant due to a one-time bug in the GUI, and recruited a new participant to redo the excluded experiment.' Final dataset: 60 valid participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The system uses GPT-4 API calls but no cost, latency, or token consumption figures are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware specified: 'Lenovo ThinkPad laptop with an external mouse and an Intel(R) Core(TM) Ultra 9 185H 2.30 GHz CPU' (Section V-F). Each experiment was 15 minutes per task. Total: 60 participants × 3 tasks × 15 min = 45 hours of experiment time."
       }
     }
   },
   "claims": [
     {
       "claim": "BETR-GUI with full AI assistant enables users to perform significantly better than manual-only programming",
       "evidence": "Table V: FULL vs MANUAL_ONLY b=61.05, p<.001. Mean scores 91.14 vs 30.24 (Table III). LMM R²=0.62.",
       "supported": "strong"
     },
     {
       "claim": "Removing BO or GP individually does not significantly degrade performance compared to FULL",
       "evidence": "Table V: FULL vs NO_BO b=1.71, p=.999; FULL vs NO_GP b=8.52, p=.807.",
       "supported": "strong"
     },
     {
       "claim": "The LLM and planner are the critical AI components — removing either eliminates the advantage over manual-only",
       "evidence": "Table V: NO_LLM vs MANUAL_ONLY b=-8.83, p=.783; NO_PLANNER vs MANUAL_ONLY b=-13.32, p=.358 (both non-significant).",
       "supported": "strong"
     },
     {
       "claim": "Humans using the full system outperform the AI assistant running alone",
       "evidence": "Table VI: FULL vs NO_HUMAN b=3.08, p<.01. Mean scores 91.1 vs 88.06. R²=0.04.",
       "supported": "moderate"
     },
     {
       "claim": "System usability follows the same pattern as task performance across GUI variants",
       "evidence": "Table IX shows same grouping: FULL/NO_BO/NO_GP significantly higher SUS than NO_LLM/NO_PLANNER/MANUAL_ONLY.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "ABB author affiliations with potential commercial interest",
       "detail": "Multiple authors are affiliated with ABB Robotics and ABB Corporate Research. The BETR-GUI tool could have commercial value for ABB's robot programming products, but no competing interests statement is included."
     },
     {
       "flag": "Gender imbalance in participants",
       "detail": "50 male vs 10 female participants (83% male). The paper does not discuss whether this imbalance limits generalizability or whether gender was included as a covariate."
     },
     {
       "flag": "Small effect size for human vs AI-only comparison",
       "detail": "The FULL vs NO_HUMAN comparison has R²=0.04, a very small effect size (3.08 point difference on a 0-100 scale). While statistically significant, the practical significance is questionable."
     },
     {
       "flag": "Unequal cell sizes across ablation conditions",
       "detail": "FULL and MANUAL_ONLY had 60 observations each while ablation conditions had only 15 each (Table I). The mixed model accounts for this but statistical power for ablation comparisons is lower."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the impact of early-2025 ai on experienced open-source developer productivity",
       "authors": [
         "J. Becker",
         "N. Rush",
         "E. Barnes",
         "D. Rein"
       ],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "Counter-evidence showing software developers could perform worse with AI assistants, directly motivating this study's research question."
     },
     {
       "title": "A survey of Behavior Trees in robotics and AI",
       "authors": [
         "M. Iovino",
         "E. Scukins",
         "J. Styrud",
         "P. Ögren",
         "C. Smith"
       ],
       "year": 2022,
       "relevance": "Comprehensive survey of BT methods in robotics including learning, planning, and LLM approaches relevant to AI-assisted programming."
     },
     {
       "title": "The illusion of thinking: Understanding the strengths and limitations of reasoning models via the lens of problem complexity",
       "authors": [
         "P. Shojaee",
         "I. Mirzadeh",
         "K. Alizadeh",
         "M. Horton",
         "S. Bengio",
         "M. Farajtabar"
       ],
       "year": 2025,
       "relevance": "Documents LLM limitations in complex long-horizon planning tasks, contextualizing why combining LLMs with other methods is beneficial."
     },
     {
       "title": "Automatic behavior tree expansion with llms for robotic manipulation",
       "authors": [
         "J. Styrud",
         "M. Iovino",
         "M. Norrlöf",
         "M. Björkman",
         "C. Smith"
       ],
       "year": 2025,
       "relevance": "Direct predecessor work (BETR-XP-LLM) combining LLMs with planners for BT creation, which BETR-GUI builds upon."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": [
         "ChatDev team"
       ],
       "year": 2023,
       "relevance": "Multi-agent LLM system for software development, relevant to AI-assisted programming paradigms."
     },
     {
       "title": "LLM+P: Empowering large language models with optimal planning proficiency",
       "authors": [
         "B. Liu"
       ],
       "year": 2023,
       "arxiv_id": "2304.11477",
       "relevance": "Combines LLMs with PDDL planners, a key technique used in BETR-GUI's AI assistant pipeline."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The BETR-GUI tool is niche to robotics behavior tree programming, not broadly applicable to most developers' daily work."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that humans+AI outperform AI alone is mildly interesting but largely expected; the ablation showing LLM and planner are critical while GP/BO are not is a minor surprise."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns are raised; this is a constructive tool for robot programming."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no company claims challenged, no conflict angle."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is on GitHub but requires Unity simulation, PyQt5, and GPT-4 API access — significant setup effort."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from KTH, ABB, and ETH — recognized in robotics but not household names in the broader tech community."
     }
   }
 }