ai-research-survey

scan.json (25617B)

---

 {
   "paper": {
     "title": "HyperAgent: Generalist Software Engineering Agents to Solve Coding Tasks at Scale",
     "authors": ["Huy N. Phan", "Tien N. Nguyen", "Phong X. Nguyen", "Nghi D. Q. Bui"],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2409.16299",
     "doi": "10.48550/arXiv.2409.16299"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "HyperAgent is a multi-agent system with four specialized agents (Planner, Navigator, Editor, Executor) that achieves competitive or state-of-the-art results across SWE-Bench (33.00% Verified, 26.00% Lite), RepoExec (53.33% Pass@5), and Defects4J (59.70% Acc@1 fault localization, 22.9% correct repair rate). Ablation studies show each agent role contributes meaningfully, with Navigator removal causing the largest performance drop. The system demonstrates cross-task and cross-language generality (Python and Java) with a single framework.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided: https://github.com/FSoft-AI4Code/HyperAgent (stated on page 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: SWE-Bench, RepoExec, and Defects4J. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is described in the paper. Only model names are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub repo is linked but the paper itself does not include reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 33.00%, 26.00%) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims HyperAgent 'outperforms' or 'surpasses' baselines based solely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context (e.g., 'surpassing AutoFL by 8.7 percentage points (51.00%)' in Section 5.3.2), providing enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. The benchmark sizes are taken as given (SWE-Bench Lite 300, Verified 500, RepoExec 355, Defects4J 353/395/440) without discussing whether these are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experiment. Results appear to be single-run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines compared across all three tasks: SWE-Agent, AutoCodeRover, Agentless for SWE-Bench; RAG baselines and CodeLlama for RepoExec; DeepFL, AutoFL, Grace, DStar, Ochiai, RepairAgent, SelfAPR, ITER for Defects4J."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods like SWE-Agent (2024), Agentless (2024), AutoCodeRover (2024), and RepairAgent (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 presents ablation studies on agent roles (removing Navigator, Editor, Executor individually) and Section 6.2 presents ablation on tool design choices, both on SWE-bench Tiny."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: resolved percentage, average time cost, average dollar cost for SWE-Bench; pass@1 and pass@5 for RepoExec; acc@1 for fault localization; plausible and correct fixes for program repair."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs. All evaluations are automated (test suite pass/fail, AST matching for repair correctness)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Uses established held-out benchmarks: SWE-Bench Verified (manually validated by professional annotators), SWE-Bench Lite, RepoExec, and Defects4J."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 (Appendix A.6) provides per-project breakdown for Defects4J repair across 13 projects. Multiple HyperAgent configurations are compared separately."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 provides error analysis with categorized failure types (edit failed loop, early exit, hallucination, timeout) shown in Figure 3."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "HyperAgent-Lite-2 and Full-3 (open-source Llama configurations) show substantially lower performance (11-18%), honestly reported. Ablations show performance drops. Section 6.4 discusses hallucination problems."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'state-of-the-art results' and 'surpassing strong baselines' are supported by Tables 1-4 showing competitive or best results across benchmarks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies (Section 6.1, Table 5) use controlled single-variable manipulation to support causal claims about component contributions. Each agent role is removed individually to measure its impact."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Generalist Software Engineering Agents' but testing is limited to three benchmarks in Python and Java. The abstract claims 'broad spectrum of SE tasks across multiple programming languages' but only two languages are tested. The paper claims to be 'the first system designed to work off-the-shelf across diverse SE tasks' which overstates what two languages and three task types demonstrate."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the observed results. For example, the paper does not discuss whether performance gains come from the multi-agent architecture or from the specific LLMs used, or whether better prompting of a single agent could achieve similar results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames benchmark performance (SWE-Bench pass rates, Defects4J repair rates) as evidence of 'generalist software engineering' capability without discussing the gap between benchmark performance and actual real-world SE ability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 7 lists 'Claude-3-Sonnet', 'Claude-3-Haiku', 'GPT-4o', 'Llama-3-70B', 'Llama-3-8B' without specific version snapshots or API dates. 'LLaMa-3.1-8B-Instruct' is mentioned for the summarizer but other models lack version specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system prompts for all four agents (Planner, Navigator, Editor, Executor) are provided in Appendix A.7, including task templates in Appendix A.1."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM hyperparameters are reported anywhere in the paper."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent architecture is described in detail in Section 3, including agent roles, communication via message queues, tool descriptions (Section 3.3, Tables 8-9, Appendix A.3), and workflow diagrams (Figures 1-2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data setup for each benchmark is documented: SWE-Bench Lite/Verified filtering criteria, RepoExec context exclusion rationale, Defects4J version selection. Section 5 and Appendix A.4 describe the evaluation setup."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. The conclusion mentions future work directions but does not discuss current limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The error analysis in Section 6.4 discusses system failures but not threats to the validity of the evaluation methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper does not bound its claims to the tested languages (Python, Java) or task types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (trajectories, logs, per-instance results) is made available for independent verification. Only aggregate results in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark datasets are well-described: SWE-Bench sourcing from 12 Python repositories (Section 5.1.1), RepoExec's 355 samples with 96.25% test coverage (Section 5.2.1), Defects4J's 353 active bugs (Section 5.3.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper uses standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: how benchmarks are set up, how patches are applied, how tests are run, and how correctness is verified (e.g., AST matching for Defects4J in Section 5.3.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: FPT Software AI Center and University of Texas at Dallas."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors are from FPT Software AI Center, a commercial entity that could benefit from demonstrating strong AI coding capabilities. No discussion of independence. No funding disclosure to assess."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Authors are from FPT Software, which has a commercial interest in AI coding tools."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the LLMs used (Claude-3, GPT-4o, Llama-3)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs may have seen SWE-Bench, RepoExec, or Defects4J data during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-Bench and Defects4J are publicly available benchmarks that predate the models' training. No discussion of contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Average cost per instance reported in Tables 1-3 (e.g., $0.45 for HyperAgent-Lite-1, $1.82 for Full-1 on SWE-Bench; $0.18 for RepoExec and fault localization)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated (total API spend, GPU hours for running all experiments). Only per-instance costs are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented without indicating whether they are from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Five configurations are tested (Table 7) but no search process is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All five configurations are reported transparently in Table 1, not just the best one. The paper does not select a single 'best' — it presents all results."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment of author-evaluation bias. The authors evaluate their own system against baselines without discussing this potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-3 report both performance and cost (time and dollar) per configuration, allowing readers to compare performance-cost tradeoffs across systems."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SWE-Bench, RepoExec, or Defects4J actually measure 'generalist software engineering' capability as claimed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares HyperAgent (its own scaffold) against SWE-Agent (different scaffold), AutoCodeRover (different scaffold), and Agentless (no scaffold), but does not discuss the scaffold confound. Performance differences could be due to scaffolding rather than the approach."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. SWE-Bench issues and Defects4J bugs may have been seen during model training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples share structural similarities with training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used."
       }
     }
   },
   "claims": [
     {
       "claim": "HyperAgent-Full-1 achieves 33.00% on SWE-Bench Verified and 26.00% on SWE-Bench Lite, outperforming Agentless+GPT-4o (24.30% Lite) and SWE-Agent+Claude 3.5 Sonnet (23.00% Lite).",
       "evidence": "Table 1, Section 5.1.2. Direct comparison of resolved instance percentages.",
       "supported": "moderate"
     },
     {
       "claim": "HyperAgent-Lite-3 achieves the highest Pass@5 (53.33%) on RepoExec without provided context, outperforming all RAG baselines and full-context models.",
       "evidence": "Table 2, Section 5.2.2. Comparison against RAG baselines and CodeLlama with full context.",
       "supported": "moderate"
     },
     {
       "claim": "HyperAgent achieves 59.70% Acc@1 on Defects4J fault localization, surpassing AutoFL (51.00%) by 8.7 percentage points.",
       "evidence": "Table 3, Section 5.3.2.",
       "supported": "moderate"
     },
     {
       "claim": "HyperAgent achieves 192 correct fixes (22.9%) on Defects4J, outperforming RepairAgent (164, 19.64%), SelfAPR (110, 13.17%), and ITER (57, 6.82%).",
       "evidence": "Table 4 and Table 10 (Appendix A.6). Per-project breakdown provided.",
       "supported": "moderate"
     },
     {
       "claim": "HyperAgent is the first system designed to work off-the-shelf across diverse SE tasks in multiple programming languages without task-specific adaptations.",
       "evidence": "Stated in Section 1 contributions. Demonstrated by evaluating on Python (SWE-Bench, RepoExec) and Java (Defects4J) benchmarks.",
       "supported": "weak"
     },
     {
       "claim": "Removing the Navigator agent causes the most substantial performance drop in ablation studies.",
       "evidence": "Table 5, Section 6.1. Full-1 drops from 27% to 19% without Navigator; Lite-1 drops from 24% to 9%.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or multiple runs reported",
       "detail": "All results are single point estimates with no error bars, confidence intervals, or multi-run statistics. LLM outputs are stochastic, so single-run results may not be reproducible."
     },
     {
       "flag": "Overclaiming generality",
       "detail": "The paper claims 'generalist' capability across 'multiple programming languages' but tests only on Python and Java, and only on three task types. The title and abstract overstate the scope of the evaluation."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section, which is unusual for a systems paper making broad claims."
     },
     {
       "flag": "Scaffold confound in comparisons",
       "detail": "Comparisons against SWE-Agent, AutoCodeRover, and Agentless conflate model differences with scaffold differences. It's unclear whether gains come from the multi-agent architecture or the underlying models."
     },
     {
       "flag": "Company affiliation without conflict disclosure",
       "detail": "Authors are from FPT Software AI Center, which has commercial interest in AI coding tools, but no conflicts of interest are disclosed."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Public benchmarks (SWE-Bench, Defects4J) predate the models used. No analysis of whether models may have seen benchmark data during training."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R. Narasimhan"],
       "year": 2023,
       "relevance": "Primary benchmark for evaluating GitHub issue resolution by coding agents."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Key baseline agent system for SWE-Bench, demonstrates importance of agent-computer interface design."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Two-stage agent pipeline for bug fixing, key baseline in SWE-Bench evaluation."
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Simplified non-agent approach that outperforms complex agent systems, important baseline."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong", "Xiawu Zheng", "Jonathan Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent system for complex software generation from requirements."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2024,
       "relevance": "Multi-agent software development system using communicative agents."
     },
     {
       "title": "AgileCoder: Dynamic Collaborative Agents for Software Development Based on Agile Methodology",
       "authors": ["Minh Huynh Nguyen", "Thang Phan Chau", "Phong X. Nguyen", "Nghi D. Q. Bui"],
       "year": 2024,
       "arxiv_id": "2406.11912",
       "relevance": "Collaborative multi-agent coding system by same research group."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "LLM-based multi-agent system for autonomous bug fixing, key baseline for Defects4J evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational work in LLM code generation evaluation."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2024,
       "relevance": "Self-reflection mechanism for language agents, relevant to agentic coding workflows."
     },
     {
       "title": "RepoExec: Evaluate Code Generation with a Repository-Level Executable Benchmark",
       "authors": ["Nam Le Hai", "Dung Manh Nguyen", "Nghi D. Q. Bui"],
       "year": 2024,
       "arxiv_id": "2406.11927",
       "relevance": "Repository-level code generation benchmark used for evaluation, emphasizes executability and correctness."
     },
     {
       "title": "MASAI: Modular Architecture for Software-Engineering AI Agents",
       "authors": ["Daman Arora", "Atharv Sonwane", "Nalin Wadhwa"],
       "year": 2024,
       "arxiv_id": "2406.11638",
       "relevance": "Modular multi-agent architecture for SE tasks, directly comparable approach."
     }
   ]
 }