ai-research-survey

scan.json (26648B)

---

 {
   "paper": {
     "title": "AgentSpawn: Adaptive Multi-Agent Collaboration Through Dynamic Spawning for Long-Horizon Code Generation",
     "authors": [
       "Igor Costa"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.07072"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or archive is provided. The conclusion explicitly says future work includes 'implementing the AgentSpawn prototype,' indicating no implementation exists yet."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper references SWE-bench and Defects4J as benchmarks, but no dataset or data release is provided. The custom refactoring task set (100 tasks) is not released anywhere."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification is provided. No requirements.txt, Dockerfile, or dependency list is mentioned. The paper proposes future implementation using Claude 3.5 Sonnet or GPT-4 but does not specify any environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The conclusion explicitly lists 'empirical validation on SWE-bench and Defects4J with proper baseline configurations' as future work, confirming the experiments were not actually run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any result. All tables show single point estimates (e.g., '+97%', '67%') with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported anywhere in the paper. Comparative claims such as 'AgentSpawn achieves +97% over GPT-4 Single' are made with no p-values, t-tests, or other tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports percentage improvements over baselines but these are presented without uncertainty or baseline absolute values for context (Table 2 uses 'baseline' without stating the actual rate), making effect sizes uninterpretable. More critically, the results are not from real experiments."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 300 SWE-bench tasks, 200 Defects4J tasks, and 100 custom tasks, but no justification is provided for these sizes. More critically, the conclusion confirms these experiments were never run."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any result. All numbers are single-point estimates with no indication of run-to-run variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "While Table 2 lists baselines (GPT-4 Single, AutoGen, CrewAI, AFLOW), the conclusion admits that 'empirical validation on SWE-bench and Defects4J with proper baseline configurations' is future work — meaning the baseline comparisons were never actually conducted."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines listed (AutoGen, CrewAI, AFLOW) are contemporary, but since the experiments were never conducted (per the conclusion), this is moot. The numbers in Table 2 are not from real experiments."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 presents an ablation study but the conclusion confirms these are not real experimental results — 'empirical validation' is listed as future work. The ablation numbers are fabricated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 proposes five metrics (task completion rate, memory overhead, spawn count, coherence violations, cost per success) but no experiments were conducted. Proposing metrics in a design is not the same as using them in an evaluation. No metrics were actually measured."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates code generation quality using automated metrics (task completion, test pass rates). Human evaluation is not applicable and not claimed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No train/test split is described. The paper uses existing benchmarks (SWE-bench, Defects4J) as test sets, but since no experiments were run, this is irrelevant. No data splitting methodology is specified."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Tables 4 and 5 present breakdowns by task difficulty (Easy/Medium/Hard) and task type (Simple refactor/Multi-file fix/Complex feature). However, since the conclusion confirms no experiments were run, these numbers are fabricated. Presenting invented data in categorized tables does not constitute providing per-category breakdowns of actual results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No actual failure cases are shown or analyzed. Section 5.4 mentions limitations but does not present any empirical failure analysis or examples of when AgentSpawn fails."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper only presents positive results (AgentSpawn outperforms all baselines on all tasks). No negative results are reported, and since no experiments were conducted, this is expected."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims '34% higher completion rates than static baselines on benchmarks like SWE-bench' and '42% memory overhead reduction.' The conclusion contradicts this by listing empirical validation as future work, meaning the abstract's empirical claims are not supported by actual experiments."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes strong causal claims (e.g., 'memory slicing reduces context overflow failures by 42%', 'adaptive spawning enables specialized expertise application') with no experimental basis. Ablation in Table 3 purports to show component contributions causally, but no experiments were conducted."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims results on 'benchmarks like SWE-bench' in the abstract without clarifying that no actual evaluation was performed. Section 5.6 also generalizes to 'document authoring, data analysis, system administration' without any evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. Given that results are fabricated, there is nothing to explain. Section 5.4 mentions hyperparameter sensitivity and domain generalization as limitations but not as alternative explanations for observed results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion mentions 'implementing the AgentSpawn prototype using Claude 3.5 Sonnet or GPT-4' as future work, confirming no specific model was used. No model versions are specified for the reported experiments."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompts are provided. Skills are described abstractly as 'parameterized prompts s = (p, θs)' with example templates, but no actual prompt text used in experiments is given — and no experiments were conducted."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides hyperparameter specifications (δ=0.7, θ=0.5, weights w1-w5, max spawn depth=3, concurrent spawn limit=4, timeout=600s, relevance weights α=0.3, β=0.3, γ=0.2, δ=0.2). These are proposed rather than empirically tuned, but they are stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively describes the agentic scaffolding: five components (Memory Manager, Skill Library, Spawn Controller, Resume Coordinator, Coherence Manager), three algorithms (memory slicing, adaptive spawning, coherence protocol), and data structures (SpawnPackage, ResumePackage) in Appendix A."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing steps are described. The paper mentions using SWE-bench (300 tasks) and Defects4J (200 tasks) and a custom set (100 tasks) but provides no description of how tasks were selected, filtered, or preprocessed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 'Limitations' discusses four limitations: hyperparameter sensitivity, semantic merge complexity, domain generalization, and spawn depth scaling. This is a substantive (not just a single sentence) dedicated section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 mentions limitations but does not address threats to validity of the experimental results. Crucially, it does not acknowledge that the results are not from real experiments — the most significant validity threat in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims results in the abstract and body without clarifying that no empirical validation has been performed. The conclusion's future work list reveals the scope, but the paper presents its (non-existent) results as if they are real without clearly bounding what was and was not demonstrated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The experimental results presented in Tables 2-6 are not from actual experiments — the conclusion explicitly lists empirical validation as future work, meaning there is no underlying data to verify."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "No data collection procedure is described because no data was collected. The paper refers to existing benchmarks (SWE-bench, Defects4J) and a 'custom refactoring task set' without describing how either was prepared."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper evaluates code generation benchmarks, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented because no experiments were conducted. The conclusion confirms empirical validation is future work, making any data pipeline description inapplicable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section, no grant information, and no mention of institutional support."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author lists 'AutoHand Evolve' as affiliation with the email igor@autohand.ai. The affiliation is stated on the title page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since no funding is disclosed, it cannot be determined whether the funder is independent. The author is affiliated with AutoHand Evolve, a company that would presumably benefit from the proposed architecture being seen as successful."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interests declaration is present anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No model training cutoff is stated. The conclusion mentions 'implementing using Claude 3.5 Sonnet or GPT-4' as future work, and no specific model with a training cutoff was used in experiments."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test overlap is present. SWE-bench is a public benchmark that LLMs may have been trained on; this is not addressed. No actual experiments were conducted."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench (used as benchmark) was published before the training cutoffs of many current LLMs. The paper does not address contamination risk. No actual experiments were conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study evaluates automated code generation benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or randomized controlled trial design are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 6 presents cost figures ($2.10 vs. $2.30 per success, API call counts, timing) and Section 5.3 discusses O(n·ℓ) cost scaling. However, these are hypothetical projections from experiments that were never conducted (per the conclusion's future work list), not measured inference costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget for the experiments is stated. Since no experiments were conducted, there is no actual compute budget to report. Table 6 shows per-task estimates but not total experimental budget."
       }
     }
   },
   "claims": [
     {
       "claim": "AgentSpawn achieves 34% higher task completion rates than static baselines on benchmarks like SWE-bench.",
       "evidence": "Abstract and Section 6 (Conclusion). Table 2 shows +97% over GPT-4 Single on SWE-bench. However, the conclusion (Section 6) explicitly lists 'empirical validation on SWE-bench and Defects4J with proper baseline configurations' as future work item (2), contradicting the claim that these results were obtained.",
       "supported": "unsupported"
     },
     {
       "claim": "AgentSpawn reduces memory overhead by 42% through selective memory slicing.",
       "evidence": "Abstract, Section 3.2, Table 5. The 42% figure is derived from the algorithm design (87K → 51K tokens for multi-file fix tasks shown in Figure 2). This appears to be a calculated/simulated figure, not measured from real experiments.",
       "supported": "weak"
     },
     {
       "claim": "Adaptive spawning (Algorithm 2) provides the largest contribution to performance gains (+18-23% across benchmarks).",
       "evidence": "Table 3 ablation study. Since no experiments were conducted (conclusion lists empirical validation as future work), these ablation numbers are not from real experiments.",
       "supported": "unsupported"
     },
     {
       "claim": "73% of concurrent modification conflicts are resolvable via semantic (LLM-based) merge.",
       "evidence": "Section 3.6, Equation 6, Figure 4. Stated as 'based on analysis of typical conflict patterns in multi-file refactoring tasks.' No empirical source or prior work is cited for this figure, and it is presented as a design assumption.",
       "supported": "weak"
     },
     {
       "claim": "AgentSpawn achieves lower cost per successful completion ($2.10 vs. $2.30) despite higher per-task cost.",
       "evidence": "Table 6 cost-benefit analysis. No actual experiments were run; these numbers are projections. The conclusion confirms empirical validation is future work.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": [
     "theoretical",
     "benchmark-eval"
   ],
   "key_findings": "AgentSpawn proposes a multi-agent architecture enabling dynamic agent spawning based on runtime complexity metrics, with automatic memory slicing, skill inheritance, and coherence management for concurrent agents. The paper presents experimental results showing substantial gains over baselines (34-97% improvement on SWE-bench, Defects4J, and custom tasks) and a 42% memory reduction through selective slicing. Critically, the conclusion reveals that actual empirical validation is future work, meaning the reported experimental results were not obtained from real experiments. The paper is a theoretical design proposal with simulated or hypothetical results presented as if they were empirical findings.",
   "red_flags": [
     {
       "flag": "Fabricated experimental results presented as real",
       "detail": "The paper presents Tables 2-6 as experimental results (complete with specific numbers like '+97% on SWE-bench', '$2.10 cost per success') while the conclusion explicitly lists 'empirical validation on SWE-bench and Defects4J with proper baseline configurations' as future work item (2). This is a fundamental integrity issue: the paper reads as if experiments were conducted when they were not."
     },
     {
       "flag": "No implementation exists",
       "detail": "The conclusion states that future work includes 'implementing the AgentSpawn prototype using Claude 3.5 Sonnet or GPT-4.' An architectural paper presenting quantitative benchmarking results without any actual implementation is presenting simulated or invented results."
     },
     {
       "flag": "Claims dramatically outrun evidence",
       "detail": "The abstract claims '34% higher completion rates than static baselines on benchmarks like SWE-bench' as an established finding, but this is actually a design goal unsupported by any empirical evidence. The gap between what is claimed and what exists (only architectural design with algorithms) is extreme."
     },
     {
       "flag": "No statistical validation",
       "detail": "No confidence intervals, error bars, p-values, or variance measures are provided for any result. Even if experiments had been conducted, reporting single point estimates for systems with significant stochastic variation (LLM outputs, complex task outcomes) would be methodologically inadequate."
     },
     {
       "flag": "Conflict resolution statistics of uncertain provenance",
       "detail": "The paper states '73% of conflicts resolvable via semantic merge' (Figure 4, Equation 6) based on 'analysis of typical conflict patterns in multi-file refactoring tasks' with no citation, source dataset, or methodology described. This figure appears to be asserted without evidence."
     },
     {
       "flag": "Single author, no institutional affiliation with established research track record",
       "detail": "The paper is authored by a single researcher at 'AutoHand Evolve' with no institutional academic affiliation. No competing interests, funding, or peer review history is disclosed."
     },
     {
       "flag": "No code or artifacts released",
       "detail": "No implementation, code, data, or reproducible artifact of any kind is provided. Since the system was never implemented, reproducibility is impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "A survey on code generation with llm-based agents",
       "authors": [
         "Yihong Dong",
         "Xue Jiang",
         "Jiaru Qian",
         "Tian Wang",
         "Kechi Zhang",
         "Zhi Jin",
         "Ge Li"
       ],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey of LLM-based agents for code generation, directly relevant to the survey scope."
     },
     {
       "title": "A comprehensive survey of self-evolving ai agents: A new paradigm bridging foundation models and lifelong agentic systems",
       "authors": [
         "Jinyuan Fang",
         "Yanwen Peng",
         "Xi Zhang"
       ],
       "year": 2025,
       "arxiv_id": "2508.07407",
       "relevance": "Survey of self-evolving AI agents, relevant to agentic AI methodology evaluation."
     },
     {
       "title": "A survey of self-evolving agents: On path to artificial super intelligence",
       "authors": [
         "Huan-ang Gao",
         "Jiayi Geng",
         "Wenyue Hua"
       ],
       "year": 2025,
       "arxiv_id": "2507.21046",
       "relevance": "Survey of self-evolving agent architectures, relevant to the agentic AI research landscape."
     },
     {
       "title": "MetaGPT: Meta programming for multi-agent collaborative framework",
       "authors": [
         "Sirui Hong",
         "Xiawu Zheng",
         "Jonathan Chen"
       ],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework for software development with role-based specialization, core related work for code generation agents."
     },
     {
       "title": "Communicative agents for software development",
       "authors": [
         "Chen Qian",
         "Xin Cong",
         "Cheng Yang"
       ],
       "year": 2023,
       "arxiv_id": "2307.07924",
       "relevance": "ChatDev: multi-agent system for software development, directly relevant to multi-agent code generation evaluation."
     },
     {
       "title": "MemGPT: Towards LLMs as operating systems",
       "authors": [
         "Charles Packer",
         "Vivian Fang",
         "Shishir G Patil",
         "Kevin Wooders",
         "Ion Stoica"
       ],
       "year": 2023,
       "arxiv_id": "2310.08560",
       "relevance": "Hierarchical memory management for LLM agents, foundational work for long-context agent systems."
     },
     {
       "title": "AFLOW: Automating agentic workflow generation",
       "authors": [
         "Jiayi Zhang",
         "Jinyu Xiang",
         "Zhaoyang Yu"
       ],
       "year": 2025,
       "relevance": "Workflow graph optimization for agentic systems, used as baseline comparison in this paper."
     },
     {
       "title": "Difficulty-aware agentic orchestration for query-specific multi-agent workflows",
       "authors": [
         "Jinwei Su",
         "Qizhen Lan",
         "Yinghui Xia"
       ],
       "year": 2025,
       "arxiv_id": "2509.11079",
       "relevance": "Dynamic agent composition based on query difficulty, directly related to adaptive multi-agent orchestration."
     },
     {
       "title": "Multi-agent collaboration mechanisms: A survey of LLMs",
       "authors": [
         "Khanh-Tung Tran",
         "Dung Dao",
         "Minh-Duong Nguyen"
       ],
       "year": 2025,
       "arxiv_id": "2501.06322",
       "relevance": "Survey of multi-agent collaboration mechanisms for LLMs, highly relevant to survey scope."
     },
     {
       "title": "A-MEM: Agentic memory for LLM agents",
       "authors": [
         "Wujiang Xu",
         "Zujie Liang",
         "Kai Mei"
       ],
       "year": 2025,
       "arxiv_id": "2502.12110",
       "relevance": "Memory architecture for LLM agents using Zettelkasten-style interconnected knowledge, relevant to memory management in agentic systems."
     },
     {
       "title": "Plan-and-act: Improving planning of agents for long-horizon tasks",
       "authors": [
         "Lutfi Eren Erdogan",
         "Nicholas Lee",
         "Sehoon Kim"
       ],
       "year": 2025,
       "arxiv_id": "2503.09572",
       "relevance": "Long-horizon planning for agents, directly relevant to the code generation domain studied."
     },
     {
       "title": "Ai agentic programming: A survey of techniques, challenges, and opportunities",
       "authors": [
         "Huanting Wang",
         "Jingzhi Gong",
         "Huawei Zhang",
         "Jie Xu",
         "Zheng Wang"
       ],
       "year": 2025,
       "arxiv_id": "2508.11126",
       "relevance": "Survey of AI agentic programming techniques, directly relevant to survey scope."
     },
     {
       "title": "When single-agent with skills replace multi-agent systems and when they fail",
       "authors": [
         "Xiaoxiao Li"
       ],
       "year": 2026,
       "arxiv_id": "2601.04748",
       "relevance": "Analysis of when single-agent vs. multi-agent approaches are appropriate, highly relevant to multi-agent system evaluation."
     }
   ]
 }