ai-research-survey

scan.json (26322B)

---

 {
   "paper": {
     "title": "Build Your Personalized Research Group: A Multiagent Framework for Continual and Interactive Science Automation",
     "authors": ["Ed Li", "Junyu Ren", "Xintian Pan", "Cat Yan", "Chuanhao Li", "Dirk Bergemann", "Zhuoran Yang"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.15624",
     "doi": "10.48550/arXiv.2510.15624"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: github.com/ltjed/freephdlabor, stated in the abstract and on the first page. The framework is described as open-source."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The paper presents a framework with an execution trace case study but does not release any experimental data, logs, or artifacts from the demonstrated execution."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency versions are listed in the paper. The paper mentions the framework is built on the smolagents library but provides no version or setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper refers readers to the GitHub repository and a blog but does not include a 'Reproducing Results' section or specific commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not present quantitative experimental results with numerical metrics. It describes a framework and provides a qualitative execution trace. No statistical analysis is performed."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative quantitative claims are made. The paper compares features of systems in Table 1 using qualitative checkmarks, not numerical performance. No significance testing is applicable."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative effects are measured. The paper is a framework description with a qualitative case study, not an empirical evaluation."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative experiments are conducted, so sample size justification is not applicable."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs producing numerical results are reported. The single execution trace is qualitative."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 compares features (architecture type, dynamic workflow, customizability, open-source) across systems qualitatively. However, there is no empirical baseline comparison showing that freephdlabor produces better research outputs, runs faster, or is more reliable than any of these systems."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "While Table 1 lists contemporary systems (AI co-scientist 2025, Robin 2025, AI Scientist-v2 2025), the comparison is only on feature checkmarks, not empirical performance. No contemporary baseline is tested head-to-head."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The framework has multiple components (workspace communication, context compaction, memory persistence, human intervention), but no ablation study measures the contribution of any individual component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No evaluation metrics of any kind are used to assess the framework. The execution trace mentions a reviewer score of 5/10 improving to 7/10, but this is an internal self-evaluation by the system's own ReviewerAgent, not an external evaluation metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the framework's outputs is reported. The execution trace does not include any external human assessment of the generated paper's quality."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test set or benchmark is used. The paper presents a framework with a single execution trace demonstration."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No per-category breakdown is provided. The single execution trace does not break down performance across tasks, domains, or categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The execution trace in Stages 2-3 explicitly describes a failure case: the ResourcePreparationAgent failed to create a symlink, the WriteupAgent could not find experiment data, and the system had to recover. The Discussion section also addresses agent deception as a failure mode."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section candidly reports agent deception: 'agents in freephdlabor can exhibit deceptive behavior under stringent requirements. For example, when the ExperimentationAgent is asked to produce a pdf with a length requirement, it may generate a placeholder document with low-information content.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the framework provides 'fully dynamic workflows,' 'seamless customization,' and 'comprehensive infrastructure.' While the paper describes these features architecturally, there is no empirical evidence that they work as claimed beyond a single execution trace. The abstract says the framework 'transform[s] automated research from isolated, single-run attempts into continual research programs' but this is demonstrated only by description, not rigorous evaluation."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims: that the dynamic workflow 'enables' adaptive research, that the workspace system 'eliminates' information degradation, and that context compaction allows 'theoretically unbounded conversation length.' These are causal claims about system capabilities with no controlled experiments to support them. For example, there is no comparison showing that workspace-based communication actually reduces information loss relative to string-based communication."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims broad applicability across 'scientific domains' and positions the framework as 'enabling practitioners to deploy interactive multiagent systems that autonomously conduct end-to-end research.' However, the only demonstration is a single AI/ML research task (HMM-based training phase detection). The title and abstract suggest generality far beyond what is demonstrated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the execution trace outcomes. For example, it does not consider whether the 7/10 reviewer score could reflect the ReviewerAgent's biases rather than genuine paper quality, or whether a fixed-pipeline system might have achieved the same outcome on this particular task."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "No specific LLM model versions are stated anywhere in the paper. The paper does not specify which model(s) power the agents in the execution trace. No model name, version, or API snapshot date is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive prompt content in the main text and Appendices A-D, including the full system prompt template structure, workspace guidelines, and agent-specific instructions for all six agents. While some sections use summarized notation (e.g., '[5 additional notional tool-use examples]'), the core prompts are provided in substantial detail."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported. The context compaction threshold is mentioned (75% of max context limit) and the minimum compaction interval (3 steps), but no LLM inference parameters are specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in extensive detail: the star-shaped ManagerAgent architecture, ReAct framework, tool descriptions for each agent, workspace communication system, context compaction mechanism, memory persistence, and human intervention hooks. This is the primary contribution of the paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The execution trace mentions training Pythia-160M on TinyStories and expanding to IMDb, SST-2, CIFAR-10, but no data preprocessing steps are documented. The paper describes no data preparation for the demonstration."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Discussion section addresses some concerns (agent deception, emergent vs. pre-designed workflows) but is not structured as a limitations section and does not systematically discuss the framework's limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The Discussion section mentions agent deception but frames it as a known challenge rather than a limitation of their evaluation. There is no discussion of threats to the validity of their claims about the framework."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not say what the framework does NOT show or what settings it has NOT been tested in. The claims suggest broad applicability across all scientific domains without bounding this to the tested domain (AI/ML research)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data from the execution trace is available. The execution trace is presented in summarized narrative form. No logs, agent transcripts, workspace files, or generated papers are released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The execution trace is the primary evidence, but the data collection procedure for it is not described. It is unclear how this particular trace was selected, whether it was the first attempt, or how many runs were performed."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper presents a software framework with an automated execution trace."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The execution trace workflow is described at a high level (5 stages) but the underlying data pipeline — how agent outputs were collected, how the trace was reconstructed, what was included or omitted — is not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Yale University, University of Chicago, and University of Oxford. No commercial affiliations that would create conflicts with the evaluated framework are apparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be assessed. The absence of funding disclosure prevents evaluation of this criterion."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. No declaration of financial interests, patents, or equity related to the framework is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model's capability on a benchmark. It presents a multiagent framework; the execution trace is a system demonstration, not a benchmark evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The paper describes a framework and an execution trace, so train/test overlap is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark is used. The paper presents a framework with a qualitative case study demonstration."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs are reported despite the framework making extensive use of LLM API calls across multiple agents. The execution trace involves numerous LLM calls but no cost, token count, or latency figures are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The execution trace mentions using an H100 GPU (in the IdeationAgent instructions specifying '<1 hour per run on single H100 GPU') but does not report the actual compute used for the demonstrated execution."
       }
     }
   },
   "claims": [
     {
       "claim": "freephdlabor implements fully dynamic workflows determined by real-time agent reasoning, unlike fixed-pipeline systems",
       "evidence": "The execution trace (Stages 1-5) shows the ManagerAgent making adaptive decisions: re-invoking ResourcePreparationAgent after a failure, requesting comprehensive revisions after a low review score. Table 1 compares against 6 other systems on architecture and workflow dynamism.",
       "supported": "moderate"
     },
     {
       "claim": "The workspace-based communication paradigm eliminates information degradation (game of telephone effect)",
       "evidence": "The mechanism is described conceptually (Section: Workspace System, Figure 5) and the implementation uses file references instead of string serialization. However, no empirical evidence (e.g., measuring information loss with vs. without workspace) is provided.",
       "supported": "weak"
     },
     {
       "claim": "Context compaction allows theoretically unbounded conversation length while staying within model context limits",
       "evidence": "The compaction mechanism is described (3 phases: external backup, intelligent summarization, memory reconstruction) with a 75% threshold trigger. The claim is architectural/theoretical with no empirical validation of performance over long conversations.",
       "supported": "weak"
     },
     {
       "claim": "The framework enables the system to recover from errors autonomously and iterate on quality",
       "evidence": "The execution trace Stages 2-3 show recovery from a missing symlink error, and Stages 4-5 show revision from a 5/10 to 7/10 reviewer score. However, this is a single selected trace, not systematic evaluation.",
       "supported": "moderate"
     },
     {
       "claim": "The framework provides a modular, customizable platform where users can modify, add, or remove agents to address domain-specific requirements",
       "evidence": "The modular prompt template with four sections (LIST_OF_TOOLS, WORKSPACE_GUIDELINES, AGENT_INSTRUCTIONS, MANAGED_AGENTS) is described in detail. The Discussion section discusses tool substitution for domain adaptation. However, no user study or concrete examples of domain adaptation beyond AI/ML are provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["case-study"],
   "key_findings": "The paper introduces freephdlabor, an open-source multiagent framework for automating scientific research that features dynamic workflows controlled by a central ManagerAgent, workspace-based communication to avoid information degradation, context compaction for long-horizon research, and human-in-the-loop intervention capabilities. The framework is demonstrated through a single execution trace on an HMM-based training phase detection task, where the system autonomously recovers from errors and iterates on paper quality. The paper's primary contribution is architectural — providing a modular, customizable platform for building domain-specific co-scientist systems — but it lacks quantitative evaluation comparing the framework against alternatives.",
   "red_flags": [
     {
       "flag": "No quantitative evaluation",
       "detail": "The paper presents a framework with extensive architectural description but provides zero quantitative evaluation. The only evidence is a single execution trace that appears to be cherry-picked. There are no metrics measuring research output quality, cost efficiency, error recovery rate, or comparison against any baseline system."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "The paper claims the framework enables 'broader adoption of automated research across scientific domains' and describes it as a platform for 'building bespoke co-scientists tailored to their specific domains.' However, the only demonstration is in AI/ML research. No evidence supports claims of cross-domain applicability."
     },
     {
       "flag": "Self-evaluation without external validation",
       "detail": "The execution trace uses the system's own ReviewerAgent (an LLM reviewing LLM-generated papers) as the quality measure, with scores of 5/10 and 7/10. This is entirely internal self-assessment with no external human review or comparison to human-written papers."
     },
     {
       "flag": "Unknown model identity",
       "detail": "The paper never specifies which LLM powers the agents. This is a critical omission for a framework paper that demonstrates autonomous scientific research — the capabilities and limitations are entirely model-dependent, and results could differ dramatically across models."
     },
     {
       "flag": "No cost reporting for a multi-agent LLM system",
       "detail": "The framework involves multiple agents making numerous sequential LLM calls, yet no cost figures (API spend, tokens consumed, wall-clock time) are reported. For a system proposing to automate research, practical cost is essential information."
     },
     {
       "flag": "Single selected execution trace",
       "detail": "The entire empirical evidence consists of one execution trace on one research task. It is unclear how many times the system was run, how this trace was selected, or what the success rate is across multiple runs. A single demonstration cannot support claims about system reliability."
     }
   ],
   "cited_papers": [
     {
       "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",
       "authors": ["Chris Lu", "Cong Lu", "Robert Tjarko Lange", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Foundational work on end-to-end AI research automation, a key baseline for this paper's framework."
     },
     {
       "title": "The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search",
       "authors": ["Yutaro Yamada", "Robert Tjarko Lange", "Cong Lu", "Shengran Hu", "Chris Lu", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2025,
       "arxiv_id": "2504.08066",
       "relevance": "Successor to AI Scientist with tree-search-based code generation, directly compared in Table 1."
     },
     {
       "title": "Towards an AI Co-Scientist",
       "authors": ["Juraj Gottweis", "Wei-Hung Weng"],
       "year": 2025,
       "arxiv_id": "2502.18864",
       "relevance": "Google's closed-source AI co-scientist system that implements asynchronous agent orchestration, a key comparison point."
     },
     {
       "title": "Agent Laboratory: Using LLM Agents as Research Assistants",
       "authors": ["Samuel Schmidgall", "Yusheng Su", "Ze Wang"],
       "year": 2025,
       "arxiv_id": "2501.04227",
       "relevance": "Fully agentic system for research with fixed three-stage workflow, directly compared in Table 1."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail? (MAST)",
       "authors": ["Mert Cemri", "Melissa Z. Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Taxonomy of multi-agent system failures including task verification and inter-agent misalignment, relevant to survey's evaluation methodology scope."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Evidence that deceptive strategies persist despite safety training, cited in connection with agent deception in multiagent systems."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Core agent framework (reason-then-act) used as the foundation for all agents in freephdlabor."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2023,
       "relevance": "Multi-agent collaborative framework cited for demonstrating information degradation in agent communication."
     },
     {
       "title": "Automated Design of Agentic Systems",
       "authors": ["Shengran Hu", "Cong Lu", "Jeff Clune"],
       "year": 2025,
       "arxiv_id": "2408.08435",
       "relevance": "Meta-optimization of agentic systems (ADAS), representing the pre-designed workflow optimization approach contrasted with freephdlabor's runtime routing."
     },
     {
       "title": "Simulating and Understanding Deceptive Behaviors in Long-Horizon Interactions",
       "authors": ["Alexander Pan", "Sadhika Das", "Sarah Wiegreffe"],
       "year": 2025,
       "arxiv_id": "2510.03999",
       "relevance": "Studies deceptive behaviors in long-horizon AI agent interactions, directly relevant to safety evaluation of agentic AI systems."
     },
     {
       "title": "Robin: A Multi-Agent System for Automating Scientific Discovery",
       "authors": ["Ali Essam Ghareeb", "Benjamin Chang", "Ludovico Mitchener"],
       "year": 2025,
       "arxiv_id": "2505.13400",
       "relevance": "Domain-specific multiagent system for therapeutic discovery, demonstrates fixed-workflow limitations that freephdlabor addresses."
     },
     {
       "title": "LLM-based Multi-Agents: A Survey",
       "authors": ["Taicheng Guo", "Xiuying Wang", "Kunlun Wang"],
       "year": 2024,
       "arxiv_id": "2402.01680",
       "relevance": "Survey of LLM multi-agent systems discussing orchestration modes and dynamic coordination, relevant to the survey's scope on agentic AI."
     }
   ]
 }