ai-research-survey

scan.json (21074B)

---

 {
   "paper": {
     "title": "Automated Design of Agentic Systems",
     "authors": ["Shengran Hu", "Cong Lu", "Jeff Clune"],
     "year": 2025,
     "venue": "ICLR 2025",
     "arxiv_id": "2408.08435"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'All code is open-sourced at https://github.com/ShengranHu/ADAS.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (ARC, DROP, MGSM, MMLU, GPQA, GSM8K, GSM-Hard). The sampled validation/test splits for ARC are described but it is unclear if the exact splits are released; however the benchmarks themselves are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is mentioned in the paper. The paper mentions using GPT-4 and GPT-3.5 APIs but does not specify library versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references open-sourced code but does not include step-by-step reproduction instructions in the paper itself. Algorithmic details are in appendices but specific commands or scripts to replicate experiments are not described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1, 2, and 3 all report 95% bootstrap confidence intervals (e.g., '79.4 ± 0.8'). Figure 3a shows 95% bootstrap confidence intervals."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal significance tests (p-values, t-tests, etc.) are reported. Comparisons rely on overlapping/non-overlapping confidence intervals but no explicit statistical tests are conducted."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements with baseline context, e.g., 'improve F1 scores on reading comprehension tasks in DROP by 13.6/100' and 'accuracy rates on math tasks in MGSM by 14.4%', with baseline values visible in tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The ARC experiment uses only 20 validation and 60 test questions sampled from the easy subset. No justification for these small sample sizes is provided, nor is a power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are evaluated 5 times and reported with 95% bootstrap confidence intervals, which implicitly convey variance. Figure 3a shows median accuracy with confidence intervals across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven hand-designed baselines are compared: Chain-of-Thought, COT-SC, Self-Refine, LLM Debate, Step-back Abstraction, Quality-Diversity, Role Assignment, plus OPRO for prompt optimization."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods like Self-Refine (2024), LLM Debate (2023), OPRO (2024), and Quality-Diversity (2024). These represent current state-of-the-art agentic design patterns."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is presented that systematically removes components of Meta Agent Search (e.g., the archive, self-reflection steps, novelty encouragement) to measure their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses F1 score for DROP and accuracy for MGSM, MMLU, GPQA, GSM8K, GSM-Hard, and ARC — multiple metrics across different domains."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the discovered agents' outputs or designs is reported. All evaluation is automated via benchmark metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates validation and test sets. For ARC: '20 and 60 questions, respectively, for searching and testing.' Results are reported on held-out test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by domain (Reading Comprehension, Math, Multi-task, Science) in Table 1, and by transfer target in Table 2, and by model in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where improvements are smaller: 'While Meta Agent Search also outperforms baselines in the Multi-task and Science domains, the gap is smaller' and hypothesizes why (FM knowledge insufficient for hard science questions)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows improvement over baselines. No failed configurations, search runs that did not converge, or approaches that were tried and abandoned are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that discovered agents 'greatly outperform state-of-the-art hand-designed agents' and 'maintain superior performance even when transferred across domains and models.' Tables 1-3 support these claims with quantitative results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implicitly makes causal claims that Meta Agent Search causes the performance improvements, but the search uses the same validation data for evaluation and selection. No ablation isolates which aspects of the algorithm drive improvements vs. simply more compute/API calls."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Automated Design of Agentic Systems' is broad, but experiments are limited to single-step QA tasks using GPT-3.5/GPT-4. The paper acknowledges this in future work ('Currently, we only evaluate Meta Agent Search on single-step QA tasks') but the title and abstract do not bound claims to this setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations such as whether the improvements come from simply using more API calls (discovered agents make multiple FM queries vs. single-query baselines), or whether prompt optimization with equal compute budget would close the gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4 (OpenAI, 2024)' and 'GPT-3.5 (OpenAI, 2022)' without specifying exact model versions (e.g., gpt-4-0613, gpt-3.5-turbo-0125). Claude-Haiku and Claude-Sonnet are also used without version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The prompt and more details are presented in Appendix B' and provides framework code in Appendix C. The meta agent prompt and baseline implementations are provided in appendices."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other sampling parameters are reported for the LLM API calls. The number of iterations (25 for ARC, 30 for other domains) is stated but API-level hyperparameters are missing."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the meta agent search framework in detail: the archive mechanism, self-reflection steps, evaluation loop, and code-based agent definition. Appendices provide pseudocode and framework code."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "For ARC: 'we sample our data from questions with grid dimensions ≤5×5 in the Public Training Set (Easy). We sample a validation set and a test set with 20 and 60 questions.' Dataset details for other benchmarks reference appendices."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion and Conclusion section (Section 6) includes safety considerations and extensive future work discussion that implicitly acknowledges current limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The future work section lists extensions but does not address specific methodological concerns like small sample sizes, compute fairness of comparisons, or potential overfitting to validation sets."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The future work section explicitly states 'Currently, we only evaluate Meta Agent Search on single-step QA tasks in this paper' and lists several dimensions not tested (multi-step tasks, multi-objective optimization, etc.)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental logs, per-question results, or intermediate search archives are made available. Only aggregate results in tables are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data sources are well-described: publicly available benchmarks with specific sampling criteria (ARC grid dimensions ≤5×5 from Public Training Set Easy, standard splits for other benchmarks)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; this is a benchmark evaluation study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from sampling ARC questions to evaluation is described. The meta agent search process (iterate, evaluate, add to archive) is documented with pseudocode in Appendix H."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments list funding: 'Vector Institute, the Canada CIFAR AI Chairs program, grants from Schmidt Futures and Open Philanthropy, an NSERC Discovery Grant, and a generous donation from Rafael Cosman.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of British Columbia, Vector Institute, Canada CIFAR AI Chair."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (Vector Institute, CIFAR, Schmidt Futures, Open Philanthropy, NSERC) are research-oriented organizations without a direct financial stake in whether Meta Agent Search outperforms baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 and GPT-3.5 on benchmarks like MMLU, GSM8K, and DROP without stating the models' training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (especially MMLU, GSM8K, DROP) appeared in GPT-4/GPT-3.5 training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmarks like MMLU (2021), GSM8K (2021), and DROP (2019) were published before GPT-4's training cutoff. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time per experiment are reported. The discovered agents make multiple LLM calls per example but the cost is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend or compute budget is stated. The paper runs 25-30 iterations of meta agent search with GPT-4 plus evaluation of discovered agents, but total cost is not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Meta Agent Search discovers agents that substantially outperform state-of-the-art hand-designed agents across multiple domains.",
       "evidence": "Table 1 shows improvements of 13.6 F1 on DROP and 14.4% accuracy on MGSM over the best baselines. Tables 2 and 3 show consistent outperformance in transfer settings.",
       "supported": "strong"
     },
     {
       "claim": "Discovered agents transfer well across both domains and models.",
       "evidence": "Table 2 shows agents discovered on MGSM outperform baselines when transferred to GSM8K (+25.9%), GSM-Hard (+13.2%), and non-math domains. Table 3 shows GPT-3.5-discovered agents outperform baselines on Claude-Haiku, GPT-4, and Claude-Sonnet.",
       "supported": "strong"
     },
     {
       "claim": "Code-based search space is superior to prompt-only optimization for discovering agentic systems.",
       "evidence": "Table 1 compares Meta Agent Search against OPRO (prompt optimization). Meta Agent Search outperforms OPRO across all four domains.",
       "supported": "moderate"
     },
     {
       "claim": "Meta Agent Search progressively discovers better agents through an archive of stepping stones.",
       "evidence": "Figure 3a shows progressive improvement over 25 iterations on ARC, with annotations showing how design patterns build on previous discoveries.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Meta Agent Search, an algorithm that uses a meta agent to iteratively program new agentic systems in code, discovers agents that outperform hand-designed baselines across reading comprehension (DROP +13.6 F1), math (MGSM +14.4%), and other domains. The discovered agents transfer successfully across both domains (math to reading comprehension) and models (GPT-3.5 to Claude-Sonnet), suggesting the search discovers generalizable design patterns rather than task-specific tricks. The code-based search space outperforms prompt-only optimization (OPRO) across all tested domains.",
   "red_flags": [
     {
       "flag": "Compute fairness not addressed",
       "detail": "Discovered agents typically make multiple LLM calls per example (e.g., 5 COTs + critics + refinement), while baselines like Chain-of-Thought use a single call. No cost-controlled comparison is provided, so improvements may partly reflect increased compute rather than better design."
     },
     {
       "flag": "No ablation study",
       "detail": "The contribution of individual components of Meta Agent Search (archive, self-reflection, novelty encouragement) is not isolated. It is unclear what drives the algorithm's success."
     },
     {
       "flag": "Small and filtered evaluation set for ARC",
       "detail": "ARC experiments use only 60 test questions sampled from the easiest subset (grid dimensions ≤5×5), which may not represent the full ARC challenge difficulty."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "Multiple benchmarks (MMLU, GSM8K, DROP) predate GPT-4's training. No contamination analysis is provided despite this being a known concern for these benchmarks."
     },
     {
       "flag": "No cost reporting",
       "detail": "The total cost of running Meta Agent Search (30 iterations with GPT-4 as meta agent + evaluation of all discovered agents) is not reported, making practical feasibility difficult to assess."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R. Narasimhan"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM agents on real-world software engineering tasks."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei"],
       "year": 2022,
       "relevance": "Foundational prompting technique used as baseline and building block in agentic systems."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn"],
       "year": 2023,
       "relevance": "Self-reflection technique for LLM agents, used as baseline in this paper."
     },
     {
       "title": "DSPy: Compiling Declarative Language Model Calls into State-of-the-Art Pipelines",
       "authors": ["Omar Khattab"],
       "year": 2024,
       "relevance": "Framework for optimizing LLM pipelines, an alternative ADAS approach discussed as related work."
     },
     {
       "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",
       "authors": ["Cong Lu"],
       "year": 2024,
       "relevance": "Automated research pipeline using LLMs, closely related to automated agent design."
     },
     {
       "title": "FunSearch: Making New Discoveries in Mathematical Sciences Using Large Language Models",
       "authors": ["Bernardino Romera-Paredes"],
       "year": 2024,
       "relevance": "Uses FMs to discover optimization algorithms in code, same paradigm as ADAS code-space search."
     },
     {
       "title": "Improving Factuality and Reasoning in Language Models through Multiagent Debate",
       "authors": ["Yilun Du"],
       "year": 2023,
       "relevance": "Multi-agent debate technique used as baseline, relevant to agentic system design patterns."
     },
     {
       "title": "Large Language Models as Optimizers",
       "authors": ["Chengrun Yang"],
       "year": 2024,
       "relevance": "OPRO prompt optimization method used as baseline comparison for ADAS."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong"],
       "year": 2023,
       "relevance": "Multi-agent framework incorporating organizational structures, relevant to agentic system design."
     },
     {
       "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
       "authors": ["Guanzhi Wang"],
       "year": 2023,
       "relevance": "Embodied agent that develops new skills in code, related to code-based agent design."
     }
   ]
 }