scan-v4.json (26718B)
1 { 2 "scan_version": 4, 3 "paper_type": "empirical", 4 "paper": { 5 "title": "Design and Evaluation of an Assisted Programming Interface for Behavior Trees in Robotics", 6 "authors": [ 7 "J. Styrud", 8 "Matteo Iovino", 9 "Rebecca Stower", 10 "Mart Kartašev", 11 "Mikael Norrlöf", 12 "Mårten Björkman", 13 "Christian Smith" 14 ], 15 "year": 2026, 16 "venue": "arXiv", 17 "arxiv_id": "2602.09772", 18 "doi": null 19 }, 20 "checklist": { 21 "claims_and_evidence": { 22 "abstract_claims_supported": { 23 "applies": true, 24 "answer": true, 25 "justification": "Abstract claims that BETR-GUI 'enables users to perform better' and 'humans using the full variant perform better than the AI assistant running on its own' are both supported by the statistical results in Section VI (Tables V and VI).", 26 "source": "opus" 27 }, 28 "causal_claims_justified": { 29 "applies": true, 30 "answer": true, 31 "justification": "Causal claims ('enables users to perform better') are justified by the RCT design with randomized assignment, counterbalanced conditions, and controlled experiments (Section V-B). Mixed-model analysis accounts for individual differences.", 32 "source": "opus" 33 }, 34 "generalization_bounded": { 35 "applies": true, 36 "answer": false, 37 "justification": "Section VIII acknowledges tasks are 'highly simplified compared to actual robot applications' but the title and abstract frame results broadly as 'Assisted Programming Interface for Behavior Trees in Robotics' without bounding to the specific simplified scenarios and participant population tested.", 38 "source": "opus" 39 }, 40 "alternative_explanations_discussed": { 41 "applies": true, 42 "answer": true, 43 "justification": "Section VII discusses multiple alternative explanations: users not understanding how to use AI features, planner solving most of the task leaving little for learning algorithms, short experiment time disadvantaging learning methods, and trust issues with the AI assistant.", 44 "source": "opus" 45 }, 46 "proxy_outcome_distinction": { 47 "applies": true, 48 "answer": false, 49 "justification": "The score function (Eq. 1-4) is a composite of goal fulfillment, tree complexity, and energy penalties with manually-set weights. The paper does not discuss whether this proxy score reflects actual task quality in real robot programming. The gap between this artificial score and real-world robot programming effectiveness is not acknowledged.", 50 "source": "opus" 51 } 52 }, 53 "limitations_and_scope": { 54 "limitations_section_present": { 55 "applies": true, 56 "answer": true, 57 "justification": "Section VIII (Future Work) opens with 'The main limitation of this work is that the benchmark tasks, out of necessity, are highly simplified compared to actual robot applications' and continues with substantive discussion.", 58 "source": "opus" 59 }, 60 "threats_to_validity_specific": { 61 "applies": true, 62 "answer": true, 63 "justification": "Specific threats discussed: simplified tasks vs. real applications, short experiment time disadvantaging learning algorithms, users not understanding how to use node locking, and trust issues with the AI. Section VII discusses these specific to this study.", 64 "source": "opus" 65 }, 66 "scope_boundaries_stated": { 67 "applies": true, 68 "answer": true, 69 "justification": "Section VIII explicitly states that tasks are simplified, more time and training would be needed for realistic tasks, and that component importance depends on other components in the system.", 70 "source": "opus" 71 } 72 }, 73 "conflicts_of_interest": { 74 "funding_disclosed": { 75 "applies": true, 76 "answer": true, 77 "justification": "Funding disclosed: 'This project is supported by the Wallenberg AI, Autonomous Systems, and Software Program (WASP) funded by the Knut and Alice Wallenberg Foundation.'", 78 "source": "opus" 79 }, 80 "affiliations_disclosed": { 81 "applies": true, 82 "answer": true, 83 "justification": "Author affiliations clearly listed: ABB Robotics, KTH, ETH Zürich, ABB Corporate Research, Ericsson. Multiple authors have ABB affiliations.", 84 "source": "opus" 85 }, 86 "funder_independent_of_outcome": { 87 "applies": true, 88 "answer": true, 89 "justification": "The Wallenberg Foundation is an independent research foundation, not a product vendor. While ABB authors are listed, the funder (WASP/Wallenberg) has no direct commercial interest in the outcome.", 90 "source": "opus" 91 }, 92 "financial_interests_declared": { 93 "applies": true, 94 "answer": false, 95 "justification": "No competing interests statement is present in the paper. Multiple authors are affiliated with ABB (a robot manufacturer), and the tool could have commercial implications, but no financial interest disclosure is made.", 96 "source": "opus" 97 } 98 }, 99 "scope_and_framing": { 100 "key_terms_defined": { 101 "applies": true, 102 "answer": true, 103 "justification": "Behavior Trees are formally defined (Section II-A), the score function is specified (Equations 1–4), and GUI variants are precisely enumerated; key technical concepts are defined before use.", 104 "source": "haiku" 105 }, 106 "intended_contribution_clear": { 107 "applies": true, 108 "answer": true, 109 "justification": "Two contributions are explicitly bulleted in the introduction: the BETR-GUI tool itself and the ablative user study with 60 participants demonstrating its effectiveness.", 110 "source": "haiku" 111 }, 112 "engagement_with_prior_work": { 113 "applies": true, 114 "answer": true, 115 "justification": "Section II provides extensive engagement with prior work on BT generation methods (planning, GP, BO, LLMs, combination systems) and explicitly positions BETR-GUI as building on and integrating those prior methods.", 116 "source": "haiku" 117 } 118 } 119 }, 120 "type_checklist": { 121 "empirical": { 122 "artifacts": { 123 "code_released": { 124 "applies": true, 125 "answer": true, 126 "justification": "GitHub repository URL provided: https://github.com/jstyrud/BETR-GUI, referenced in Section III and footnotes.", 127 "source": "opus" 128 }, 129 "data_released": { 130 "applies": true, 131 "answer": true, 132 "justification": "Experimental data uploaded to the OSF repository (https://osf.io/ax5gb/overview), referenced in Sections V-D and VI.", 133 "source": "opus" 134 }, 135 "environment_specified": { 136 "applies": true, 137 "answer": false, 138 "justification": "The paper mentions Python/PyQt5 and Unity but does not provide a requirements.txt, Dockerfile, or detailed dependency list with versions.", 139 "source": "opus" 140 }, 141 "reproduction_instructions": { 142 "applies": true, 143 "answer": false, 144 "justification": "No step-by-step reproduction instructions are described in the paper. The GitHub repo is referenced but no README instructions or reproduction guide is detailed in the paper itself.", 145 "source": "opus" 146 } 147 }, 148 "statistical_methodology": { 149 "confidence_intervals_or_error_bars": { 150 "applies": true, 151 "answer": true, 152 "justification": "95% confidence intervals are reported in Tables IV, VI, and VIII for all fixed effects (e.g., 'Trial Order: 95% CI [3.0, 10.8]').", 153 "source": "opus" 154 }, 155 "significance_tests": { 156 "applies": true, 157 "answer": true, 158 "justification": "Linear mixed models with F-tests and t-tests are used throughout (Section VI). Post-hoc pairwise comparisons with Tukey adjustment for multiple comparisons are reported in Tables V and IX. Likelihood ratio test used for rankings (χ²(5) = 404.2, p < .001).", 159 "source": "opus" 160 }, 161 "effect_sizes_reported": { 162 "applies": true, 163 "answer": true, 164 "justification": "Pseudo R² values reported for models (R² = 0.62 for task performance, R² = 0.21 for SUS, R² = 0.04 for FULL vs NO_HUMAN). Regression coefficients (b values) with standard errors provided in all tables, giving effect magnitudes in the original scale.", 165 "source": "opus" 166 }, 167 "sample_size_justified": { 168 "applies": true, 169 "answer": true, 170 "justification": "Section V-E: 'The number of participants needed was determined a priori using a simulated power analysis with α = .05, suggesting 60 participants are sufficient to reach 80% power, assuming small-medium effects.'", 171 "source": "opus" 172 }, 173 "variance_reported": { 174 "applies": true, 175 "answer": true, 176 "justification": "Standard deviations reported for all GUI variant scores (Table III, e.g., 'FULL: 91.14 (9.07)'), SUS scores (Table VII), and participant demographics (Table II).", 177 "source": "opus" 178 } 179 }, 180 "evaluation_design": { 181 "baselines_included": { 182 "applies": true, 183 "answer": true, 184 "justification": "MANUAL_ONLY variant serves as baseline, described as 'largely similar to existing GUIs like Groot in capability' (Section V-A). NO_HUMAN ablation also serves as a baseline.", 185 "source": "opus" 186 }, 187 "baselines_contemporary": { 188 "applies": true, 189 "answer": true, 190 "justification": "The MANUAL_ONLY baseline is compared to Groot (2025), a contemporary commercial BT editor. The AI components build on recent prior work (2024-2025 references).", 191 "source": "opus" 192 }, 193 "ablation_study": { 194 "applies": true, 195 "answer": true, 196 "justification": "Comprehensive ablation with 6 variants: FULL, MANUAL_ONLY, NO_BO, NO_GP, NO_LLM, NO_PLANNER (Section V-A). Each ablation removes one component to measure its contribution.", 197 "source": "opus" 198 }, 199 "multiple_metrics": { 200 "applies": true, 201 "answer": true, 202 "justification": "Three metrics used: task performance score (Section V-B), System Usability Scale (Section V-F), and preference rankings (Section VI-D).", 203 "source": "opus" 204 }, 205 "human_evaluation": { 206 "applies": true, 207 "answer": true, 208 "justification": "The entire study IS a human evaluation — 60 participants used the system and provided SUS scores and rankings.", 209 "source": "opus" 210 }, 211 "held_out_test_set": { 212 "applies": false, 213 "answer": false, 214 "justification": "Not applicable — this is a user study, not a model evaluation on a dataset split.", 215 "source": "opus" 216 }, 217 "per_category_breakdown": { 218 "applies": true, 219 "answer": true, 220 "justification": "Results broken down by GUI variant (Tables III, V, VII, IX), trial number (Figures 8, 9), and task type (tested as fixed effect, found non-significant).", 221 "source": "opus" 222 }, 223 "failure_cases_discussed": { 224 "applies": true, 225 "answer": true, 226 "justification": "Section VII discusses failure modes: users not using locking functionality (74/120), users not trusting AI suggestions, and user quotes describing negative experiences with the AI assistant.", 227 "source": "opus" 228 }, 229 "negative_results_reported": { 230 "applies": true, 231 "answer": true, 232 "justification": "Several negative results reported: NO_BO and NO_GP ablations did not significantly differ from FULL; NO_LLM and NO_PLANNER did not significantly outperform MANUAL_ONLY. Hypotheses 1 and 2 only 'partially supported.'", 233 "source": "opus" 234 } 235 }, 236 "setup_transparency": { 237 "model_versions_specified": { 238 "applies": true, 239 "answer": false, 240 "justification": "Section III states 'The LLM model used in all experiments was GPT-4' without specifying a version (e.g., gpt-4-0613) or snapshot date.", 241 "source": "opus" 242 }, 243 "prompts_provided": { 244 "applies": true, 245 "answer": false, 246 "justification": "The paper references a 'slightly updated prompt' from prior work [12] but does not provide the actual prompt text. The method is described in natural language without the prompts themselves.", 247 "source": "opus" 248 }, 249 "hyperparameters_reported": { 250 "applies": true, 251 "answer": false, 252 "justification": "No temperature, top-p, or other LLM API parameters are stated. GP and BO hyperparameters are referenced as being available in the GitHub repo but not stated in the paper. Score weight values are also deferred to the repository.", 253 "source": "opus" 254 }, 255 "scaffolding_described": { 256 "applies": true, 257 "answer": true, 258 "justification": "The AI assistant pipeline is described in detail in Section III-D and Figure 2: seed BT → planner expansion → LLM error resolution → GP/BO optimization loop. Node locking, simulation evaluation, and user interaction points are all documented.", 259 "source": "opus" 260 }, 261 "data_preprocessing_documented": { 262 "applies": true, 263 "answer": true, 264 "justification": "Score normalization procedure described in Section VI-A. Experimental randomization and counterbalancing described in Section V-B. One participant excluded due to a bug and replaced (Section V-E).", 265 "source": "opus" 266 } 267 }, 268 "data_integrity": { 269 "raw_data_available": { 270 "applies": true, 271 "answer": true, 272 "justification": "Data uploaded to OSF repository (https://osf.io/ax5gb/overview). Full analyses and supplementary materials referenced as available there.", 273 "source": "opus" 274 }, 275 "data_collection_described": { 276 "applies": true, 277 "answer": true, 278 "justification": "Section V-F describes the full procedure: consent form, demographic questionnaire, 5-minute instruction video, 5-minute familiarization, three 15-minute timed trials, SUS after each variant, final ranking.", 279 "source": "opus" 280 }, 281 "recruitment_methods_described": { 282 "applies": true, 283 "answer": true, 284 "justification": "Section V-E: 'We recruited 60 participants via flyers, mailing lists, social media, and word of mouth.' Demographics, age range (20-62), gender (10F/50M), and expertise levels reported.", 285 "source": "opus" 286 }, 287 "data_pipeline_documented": { 288 "applies": true, 289 "answer": true, 290 "justification": "The GUI automatically recorded actions and scores with timestamps (Section V-F). Score normalization explained (Section VI-A). One participant excluded and replaced with reason stated (Section V-E). Statistical analysis pipeline described with model selection criteria (Section VI).", 291 "source": "opus" 292 } 293 }, 294 "contamination": { 295 "training_cutoff_stated": { 296 "applies": false, 297 "answer": false, 298 "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. GPT-4 is used as a component in the system, but the evaluation is of the human-AI system, not of GPT-4's knowledge.", 299 "source": "opus" 300 }, 301 "train_test_overlap_discussed": { 302 "applies": false, 303 "answer": false, 304 "justification": "Not evaluating a pre-trained model on a benchmark — this is a user study of a human-AI collaborative system.", 305 "source": "opus" 306 }, 307 "benchmark_contamination_addressed": { 308 "applies": false, 309 "answer": false, 310 "justification": "Not evaluating a pre-trained model on a benchmark — this is a user study of a human-AI collaborative system.", 311 "source": "opus" 312 } 313 }, 314 "human_studies": { 315 "pre_registered": { 316 "applies": true, 317 "answer": true, 318 "justification": "Section V-D: 'We pre-registered our hypotheses and planned analyses (both confirmatory and exploratory) on the Open Science Framework (OSF)' with link https://osf.io/ax5gb/overview.", 319 "source": "opus" 320 }, 321 "irb_or_ethics_approval": { 322 "applies": true, 323 "answer": false, 324 "justification": "Section V-F states 'This study followed the ethical guidelines for Sweden' but does not mention IRB or ethics board approval specifically.", 325 "source": "opus" 326 }, 327 "demographics_reported": { 328 "applies": true, 329 "answer": true, 330 "justification": "Section V-E: ages 20-62 (M=29.7, SD=8.9), 10 female/50 male, primarily engineering/CS students or professional developers. Table II reports familiarity scores across four domains.", 331 "source": "opus" 332 }, 333 "inclusion_exclusion_criteria": { 334 "applies": true, 335 "answer": false, 336 "justification": "No explicit inclusion or exclusion criteria stated. Participants described as 'primarily university students of engineering or computer science or professional software developers' but no screening criteria are mentioned.", 337 "source": "opus" 338 }, 339 "randomization_described": { 340 "applies": true, 341 "answer": true, 342 "justification": "Section V-B: 'We randomized all the experiments in advance, ensuring that all ablations occurred the same number of times, and that the order in which tasks and variants occurred was also counterbalanced.' Randomization code available on GitHub.", 343 "source": "opus" 344 }, 345 "blinding_described": { 346 "applies": true, 347 "answer": false, 348 "justification": "No mention of blinding. Participants could see which GUI variant they were using (e.g., presence/absence of AI features was obvious). No discussion of whether this awareness affected behavior.", 349 "source": "opus" 350 }, 351 "attrition_reported": { 352 "applies": true, 353 "answer": true, 354 "justification": "Section V-E: 'After the experiments, we excluded one participant due to a one-time bug in the GUI, and recruited a new participant to redo the excluded experiment.' Final dataset: 60 valid participants.", 355 "source": "opus" 356 } 357 }, 358 "cost_and_practicality": { 359 "inference_cost_reported": { 360 "applies": true, 361 "answer": false, 362 "justification": "The system uses GPT-4 API calls but no cost, latency, or token consumption figures are reported.", 363 "source": "opus" 364 }, 365 "compute_budget_stated": { 366 "applies": true, 367 "answer": true, 368 "justification": "Hardware specified: 'Lenovo ThinkPad laptop with an external mouse and an Intel(R) Core(TM) Ultra 9 185H 2.30 GHz CPU' (Section V-F). Each experiment was 15 minutes per task. Total: 60 participants × 3 tasks × 15 min = 45 hours of experiment time.", 369 "source": "opus" 370 } 371 } 372 } 373 }, 374 "claims": [ 375 { 376 "claim": "BETR-GUI with full AI assistance achieves significantly higher task scores than manual-only programming (FULL vs MANUAL_ONLY: b=61.05, p<.001 on 0–100 normalized scale)", 377 "evidence": "LMM pairwise comparison in Table V, replicated across task score, SUS, and preference rankings", 378 "supported": "strong" 379 }, 380 { 381 "claim": "Humans using the full BETR-GUI outperform AI running alone without any human editing (NO_HUMAN condition)", 382 "evidence": "LMM in Table VI: b=3.08, p<.001, though model fit is low (R²=0.04), indicating a real but modest advantage", 383 "supported": "moderate" 384 }, 385 { 386 "claim": "LLM and planning are the critical components — removing either (NO_LLM, NO_PLANNER) yields performance statistically equivalent to manual-only", 387 "evidence": "Table V: NO_LLM vs MANUAL_ONLY p=.783, NO_PLANNER vs MANUAL_ONLY p=.358; NO_LLM and NO_PLANNER differ significantly from FULL (p<.001)", 388 "supported": "strong" 389 }, 390 { 391 "claim": "Bayesian optimization and genetic programming contribute to the system but do not produce statistically significant gains over the planner+LLM core", 392 "evidence": "Table V: FULL vs NO_BO p=.999, FULL vs NO_GP p=.807 — both nonsignificant after Tukey adjustment", 393 "supported": "strong" 394 }, 395 { 396 "claim": "Task performance improved significantly with trial experience, and prior familiarity with behavior trees predicts better performance", 397 "evidence": "Table IV: trial order b=6.88, p<.001; BT familiarity b=3.99, p<.05 as fixed effects in the LMM", 398 "supported": "strong" 399 } 400 ], 401 "methodology_tags": [ 402 "rct", 403 "qualitative" 404 ], 405 "key_findings": "BETR-GUI, a behavior tree programming interface combining LLMs, automated planning, genetic programming, and Bayesian optimization, significantly outperforms manual-only programming in a pre-registered RCT with 60 participants (b=61.05 on normalized 0–100 score). The LLM and planning components are load-bearing: removing either one collapses performance to manual-only levels, while removing BO or GP has no statistically significant effect. Even with full AI assistance, human-in-the-loop participation outperforms AI running alone, suggesting the human still adds value in the 15-minute task window. However, tasks were deliberately simplified and participants often failed to leverage AI features (locking nodes used in only 74/120 AI-assisted trials), limiting generalizability to real industrial robot programming.", 406 "red_flags": [ 407 { 408 "flag": "GPT-4 version unspecified", 409 "detail": "Only 'GPT-4' is named with no API version, snapshot date, or model card reference, making exact replication of the LLM component impossible." 410 }, 411 { 412 "flag": "Prompts not disclosed", 413 "detail": "The actual prompts used for goal extraction and error resolution are referenced as 'slightly updated' from prior work but not reproduced, preventing independent replication." 414 }, 415 { 416 "flag": "Heavily male sample", 417 "detail": "50 of 60 participants (83%) identified as male; the predominantly engineering-student population may not generalize to the broader workforce of robot programmers." 418 }, 419 { 420 "flag": "Manually tuned score function", 421 "detail": "The composite score function (Equations 1–4) uses weights 'simply set manually' without extensive tuning; it is possible the weights favor approaches that the AI assistant optimizes for." 422 }, 423 { 424 "flag": "Unequal variant exposure", 425 "detail": "FULL and MANUAL_ONLY had 16–22 observations per task cell vs. 3–8 for ablation variants; while LMM accounts for this, statistical power for ablation-vs-ablation comparisons is substantially lower." 426 }, 427 { 428 "flag": "No IRB named", 429 "detail": "'Followed ethical guidelines for Sweden' is stated but no specific ethics committee, approval number, or institutional review board is identified." 430 } 431 ], 432 "cited_papers": [ 433 { 434 "title": "A survey of Behavior Trees in robotics and AI", 435 "relevance": "Foundational survey grounding BETR-GUI in the BT literature; defines the architecture and summarizes generation techniques" 436 }, 437 { 438 "title": "Automatic behavior tree expansion with LLMs for robotic manipulation (BETR-XP-LLM)", 439 "relevance": "Direct predecessor — BETR-GUI builds on this paper's LLM+planner method and reuses its prompt and error-resolution approach" 440 }, 441 { 442 "title": "Combining Planning and Learning of Behavior Trees for Robotic Assembly", 443 "relevance": "Prior ablation combining planning and GP that BETR-GUI extends with BO and LLMs" 444 }, 445 { 446 "title": "BeBOP — combining reactive planning and Bayesian optimization to solve robotic manipulation tasks", 447 "relevance": "Source of the Bayesian optimization component integrated into BETR-GUI's AI assistant" 448 }, 449 { 450 "title": "Measuring the impact of early-2025 AI on experienced open-source developer productivity", 451 "relevance": "Cited as counter-evidence that AI assistants can degrade developer performance; BETR-GUI's positive results contrast with this finding" 452 }, 453 { 454 "title": "The System Usability Scale: past, present, and future", 455 "relevance": "Methodological reference for the SUS questionnaire used to measure subjective usability across GUI variants" 456 }, 457 { 458 "title": "Fitting linear mixed-effects models using lme4", 459 "relevance": "Statistical method reference for all primary analyses in the paper" 460 }, 461 { 462 "title": "LLM-BT: Performing robotic adaptive tasks based on large language models and behavior trees", 463 "relevance": "Related work combining LLMs with BTs, positioned as an alternative approach to the one integrated in BETR-GUI" 464 } 465 ], 466 "engagement_factors": { 467 "practical_relevance": { 468 "score": 1, 469 "justification": "The BETR-GUI tool is niche to robotics behavior tree programming, not broadly applicable to most developers' daily work." 470 }, 471 "surprise_contrarian": { 472 "score": 1, 473 "justification": "The finding that humans+AI outperform AI alone is mildly interesting but largely expected; the ablation showing LLM and planner are critical while GP/BO are not is a minor surprise." 474 }, 475 "fear_safety": { 476 "score": 0, 477 "justification": "No safety, security, or risk concerns are raised; this is a constructive tool for robot programming." 478 }, 479 "drama_conflict": { 480 "score": 0, 481 "justification": "No controversy, no company claims challenged, no conflict angle." 482 }, 483 "demo_ability": { 484 "score": 1, 485 "justification": "Code is on GitHub but requires Unity simulation, PyQt5, and GPT-4 API access — significant setup effort." 486 }, 487 "brand_recognition": { 488 "score": 0, 489 "justification": "Authors are from KTH, ABB, and ETH — recognized in robotics but not household names in the broader tech community." 490 } 491 }, 492 "hn_data": { 493 "threads": [], 494 "top_points": 0, 495 "total_points": 0, 496 "total_comments": 0 497 } 498 }