ai-research-survey

scan.json (26055B)

---

 {
   "paper": {
     "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
     "authors": [
       "Xingyao Wang",
       "Boxuan Li",
       "Yufan Song",
       "Frank F. Xu",
       "Xiangru Tang",
       "Mingchen Zhuge",
       "Jiayi Pan",
       "Yueqi Song",
       "Bowen Li",
       "Jaskirat Singh",
       "Hoang H. Tran",
       "Fuqiang Li",
       "Ren Ma",
       "Mingzhang Zheng",
       "Bill Qian",
       "Yanjun Shao",
       "Niklas Muennighoff",
       "Yizhe Zhang",
       "Binyuan Hui",
       "Junyang Lin",
       "Robert Brennan",
       "Hao Peng",
       "Heng Ji",
       "Graham Neubig"
     ],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2407.16741"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/All-Hands-AI/OpenHands. Released under MIT license."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All 15 evaluation benchmarks used are publicly available standard benchmarks (SWE-Bench, WebArena, GAIA, etc.). The paper uses no proprietary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes a Docker-based sandbox environment in detail (§2.2, §F), including the runtime image build system with Dockerfiles. The open-source repository contains full environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation framework is integrated into the open-source platform (§4), and the paper describes benchmark integration in detail. The repository contains evaluation scripts for all 15 benchmarks."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars reported in any of the results tables (Tab. 3-6). All results are point estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests used despite numerous comparative claims (e.g., 'competitive performance', 'significantly better'). Comparisons are based solely on numeric differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as success rates with baseline context, e.g., CodeActAgent at 26.0% vs SWE-Agent at 18.0% on SWE-Bench Lite (Tab. 4), with absolute numbers allowing effect size computation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark subset sizes. SWE-Bench Lite uses 300 instances (cost-saving noted but not statistically justified), BIRD uses 300 samples from dev set, ML-Bench uses quarter subset — none justified statistically."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation reported. All results appear to be single-run numbers with no indication of run-to-run variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baseline comparisons included across all 15 benchmarks. Tab. 3-6 compare against SWE-Agent, AutoCodeRover, Aider, Agentless, WebArena Agent, AutoGPT, and many others."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are contemporary: SWE-Agent (2024), Agentless (2024), AutoWebGLM (2024), Moatless Tools (2024). Most baselines are from 2023-2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study. The paper tests different agents and models but does not systematically remove or modify components to measure their individual contribution to performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple benchmarks spanning different capabilities are used (software engineering, web browsing, misc. assistance). Within benchmarks, metrics like pass@k (HumanEvalFix) and success rate are used. Tab. 4 includes cost alongside accuracy."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs. All evaluations are fully automated (test suites, exact match, execution accuracy)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets are used. SWE-Bench Lite is a canonical subset, GPQA uses the diamond set, GAIA uses L1 validation set. These are established held-out sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by benchmark category (software engineering, web browsing, misc. assistance) in separate tables (Tab. 4-6). GPQA results broken down by subset (diamond, main, extended) in Tab. 7."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section A (Limitations and Future Work) discusses failure modes: agents struggle with complex tasks, long file editing, and web browsing. Specific weaknesses acknowledged."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results reported: CodeActAgent underperforms Gorilla fine-tuned model on APIBench (36.4% vs 75.0%), underperforms CC-NET on MiniWoB++ (40.8% vs 91.1%), and underperforms SWE-Agent on HumanEvalFix (79.3% vs 87.7%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are appropriately hedged: 'perform an evaluation of agents over 15 challenging tasks' (supported by §4), platform features described match implementation. No overclaimed SOTA."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CodeAct agent is 'significantly better' than non-agentic approaches on HumanEvalFix (§4.2.1) and attributes performance differences to the framework's design choices, but no controlled experiments isolate specific causal factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper frames results per-benchmark and per-model, acknowledging that the same agent demonstrates 'competitive performance' rather than claiming superiority. §4.1 explicitly notes agents are 'designed with generality in mind' rather than claiming they are the best."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for performance differences. For example, the HumanEvalFix comparison to SWE-Agent (0-shot vs 1-shot) is noted but not systematically analyzed as a confound."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses benchmark pass rates as proxies for agent capability (e.g., SWE-Bench for 'software engineering ability', WebArena for 'web browsing ability') without discussing whether these benchmarks capture the full scope of the claimed capabilities."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions provided throughout: 'gpt-4o-2024-05-13', 'claude-3-5-sonnet-20240620', 'gpt-4o-mini-2024-07-18', 'gpt-4-1106-preview' (Tab. 3-6). Version dates included."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full agent prompts provided: Browsing Agent prompt in §K with complete text, CodeAct SWE Agent prompt referenced at GitHub URL (§H). Agent implementation code in Fig. 3."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters reported for LLM API calls (temperature, top-p, max tokens). No mention of sampling settings used across experiments."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Agent scaffolding described in detail: event stream architecture (§2.1), runtime environment with bash/IPython/browser (§2.2), agent skills library (§2.3), multi-agent delegation (§2.4). Fig. 2 provides architecture overview."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark setup procedures described per-benchmark. E.g., SWE-Bench: no hint text used, BIRD: 300 samples from dev set, ML-Bench: quarter subset, BioCoder: context removed from prompts. HumanEvalFix follows Muennighoff et al. setup."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section A 'Limitations and Future Work' provides substantive discussion of multiple limitations across several paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific limitations discussed: agents struggle with complex tasks, long file editing performance is poor, web browsing needs improvement. These are specific to the system being evaluated."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statement of what the results do NOT show. The paper does not bound its generalization claims or explicitly state which settings/populations are excluded from its conclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental logs or per-instance results released. Only aggregate success rates reported in tables. Individual agent trajectories not available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Each benchmark's data collection is described by reference to the original papers, with any modifications documented (e.g., BioCoder context removal, BIRD 300-sample selection, ML-Bench quarter subset)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluations use standard automated benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: Docker sandbox setup (§F), action execution API (§2.2), benchmark integration process. The runtime workflow is diagrammed in Fig. 4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper. Multiple institutional affiliations listed but no funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: UIUC, CMU, Yale, UC Berkeley, Contextual AI, KAUST, ANU, HCMUT, Alibaba, and All Hands AI. All affiliations are transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors include affiliates of 'All Hands AI' (the commercial entity behind OpenHands) and Alibaba (whose models could be evaluated). No discussion of whether funding sources have a stake in the platform's success."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Authors from 'All Hands AI' (a company built around the OpenHands platform) have clear financial interest in the platform's success, but this is not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training cutoff dates stated for any of the LLMs used (GPT-4o, Claude 3.5 Sonnet, etc.) despite using benchmarks that may overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap for any benchmark. HumanEval (published 2021) and other benchmarks could be in training data of models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No contamination analysis despite using multiple public benchmarks (HumanEval, BIRD, SWE-Bench) with models whose training data likely includes these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. All evaluations are automated benchmark evaluations."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Average cost per instance reported for most experiments in the '$ Avg. Cost' column across Tables 4-6. E.g., SWE-Bench Lite with claude-3-5-sonnet costs $1.10 per instance."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total cost estimate provided: 'Running the complete set of 2294 instances costs $6.9k, using a conservative estimate of $3 per instance' (§4.2 footnote). Per-instance costs reported across benchmarks."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results reported. All results appear to be single-run evaluations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Number of runs not stated for any experiment. Results presented without indicating whether they are from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Agent configurations appear selected without documenting the search process."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why specific agent versions (e.g., v1.5 vs v1.8) or configurations were selected for different benchmarks. Different versions used across benchmarks without explanation."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors evaluate their own platform against competitors without acknowledging potential self-comparison bias. Baseline implementations may differ from original setups."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Cost per instance reported alongside performance for most benchmarks (Tab. 4-6), enabling compute-normalized comparisons. E.g., gpt-4o-mini at $0.01 vs gpt-4o at $1.72 per SWE-Bench instance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any of the 15 benchmarks actually measure the claimed capabilities. SWE-Bench is used as proxy for 'software engineering' ability without validity analysis."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Different scaffolds compared (SWE-Agent vs CodeActAgent vs Agentless) without isolating model vs scaffold effects. Tab. 3-4 compare systems with different scaffolding and different models simultaneously."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Many benchmarks (HumanEval 2021, BIRD 2023) predate the models used, creating potential for training data overlap."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage in benchmark evaluations."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data for any benchmark."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods applied to any benchmark evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeActAgent v1.8 with claude-3-5-sonnet achieves 26.0% on SWE-Bench Lite, competitive with specialized SWE agents.",
       "evidence": "Tab. 4 shows 26.0% vs SWE-Agent 18.0%, AutoCodeRover 19.0%, Aider 26.3%, Agentless 27.3%.",
       "supported": "strong"
     },
     {
       "claim": "The same CodeAct agent demonstrates competitive performance across three major task categories without task-specific modifications.",
       "evidence": "Tab. 3 shows CodeActAgent evaluated on SWE-Bench (26.0%), WebArena (15.3%), and GPQA (52.0%) without system prompt changes. §4.1 notes this generality.",
       "supported": "strong"
     },
     {
       "claim": "CodeActAgent fixes 79.3% of bugs in HumanEvalFix, significantly better than all non-agentic approaches.",
       "evidence": "Tab. 4: 79.3% vs StarCoder2-15B at 48.6% and DeepSeekCoder-33B at 47.5%. However, SWE-Agent (1-shot) achieves 87.7%.",
       "supported": "moderate"
     },
     {
       "claim": "CodeActAgent achieves 76.5% on ML-Bench, outperforming SWE-Agent (42.6%) and Aider (64.4%).",
       "evidence": "Tab. 4 shows these numbers. Uses gpt-4o with cost of $0.25 per instance.",
       "supported": "moderate"
     },
     {
       "claim": "OpenHands is the only framework with all 10 features compared in Tab. 1 (GUI, standardized tools, sandbox, browser, multi-agent, human-AI collaboration, AgentHub, evaluation, QC).",
       "evidence": "Tab. 1 comparison matrix shows OpenHands with checkmarks in all columns while competitors lack at least one.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OpenHands provides an open-source platform for AI software agents with a Docker-sandboxed runtime, event stream architecture, and evaluation across 15 benchmarks. The CodeActAgent achieves competitive performance as a generalist agent: 26% on SWE-Bench Lite, 15.3% on WebArena, and 52% on GPQA, all without task-specific prompt engineering. The platform reports inference costs per benchmark, showing practical deployment economics. The project has significant community traction with 32K GitHub stars and 188+ contributors.",
   "red_flags": [
     {
       "flag": "Self-evaluation by platform creators",
       "detail": "Authors from All Hands AI (the commercial entity behind OpenHands) evaluate their own platform. No independent evaluation or acknowledgment of this potential bias."
     },
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, standard deviations, or confidence intervals despite LLM outputs being stochastic."
     },
     {
       "flag": "Inconsistent agent versions across benchmarks",
       "detail": "Some benchmarks use CodeActAgent v1.5, others v1.8, with no explanation for why different versions were used or how they differ."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Uses 15 public benchmarks with commercial LLMs without any analysis of whether benchmark data appears in model training sets."
     },
     {
       "flag": "Scaffold-model confound in comparisons",
       "detail": "Comparisons in Tab. 3-4 mix scaffold differences and model differences (e.g., SWE-Agent+gpt-4-1106-preview vs CodeActAgent+claude-3-5-sonnet), making it impossible to attribute performance differences."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R. Narasimhan"],
       "year": 2024,
       "relevance": "Primary software engineering benchmark used to evaluate OpenHands agents."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Key baseline for software engineering agent evaluation; introduced ACI concept used in OpenHands."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Baseline agent approach that achieves competitive SWE-Bench performance without agentic scaffolding."
     },
     {
       "title": "Executable Code Actions Elicit Better LLM Agents",
       "authors": ["Xingyao Wang", "Yangyi Chen", "Lifan Yuan", "Yizhe Zhang", "Yunzhu Li", "Hao Peng", "Heng Ji"],
       "year": 2024,
       "relevance": "CodeAct framework underlying OpenHands' default agent architecture."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent framework compared in Tab. 1; key related work for agent framework design."
     },
     {
       "title": "MetaGPT: Meta Programming for a Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge"],
       "year": 2023,
       "relevance": "Multi-agent framework compared in Tab. 1 emphasizing standardized operating procedures."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": ["Shuyan Zhou", "Frank F. Xu", "Hao Zhu"],
       "year": 2023,
       "relevance": "Primary web browsing benchmark and baseline agent for OpenHands evaluation."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "General assistant benchmark requiring reasoning, browsing, and coding capabilities."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu", "Hao Yu"],
       "year": 2023,
       "relevance": "Multi-domain agent benchmark; OS/bash subset used for OpenHands evaluation."
     },
     {
       "title": "MINT: Evaluating LLMs in Multi-turn Interaction with Tools and Language Feedback",
       "authors": ["Xingyao Wang", "Zihan Wang", "Jiateng Liu"],
       "year": 2024,
       "relevance": "Multi-turn tool-use benchmark evaluating agent interaction with tools and feedback."
     },
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Software engineering agent baseline using code search and AST manipulation."
     },
     {
       "title": "Language agents as optimizable graphs",
       "authors": ["Mingchen Zhuge", "Wenyi Wang", "Louis Kirsch", "Francesco Faccio", "Dmitrii Khizbullin", "Jurgen Schmidhuber"],
       "year": 2024,
       "relevance": "GPTSwarm framework integrated into OpenHands AgentHub for graph-based agent optimization."
     }
   ]
 }