ai-research-survey

scan.json (26612B)

---

 {
   "paper": {
     "title": "Automating Structural Engineering Workflows with Large Language Model Agents",
     "authors": [
       "Haoran Liang",
       "Yufa Zhou",
       "Mohammad Talebi-Kalaleh",
       "Qipei Mei"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.11004"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository URL is provided at the top of the paper: https://github.com/DelosLiang/masse"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 states: 'the released dataset is reorganized but faithfully mirrors the original cases, ensuring reproducibility of system performance.' The dataset of 100 scenarios is released alongside the code repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is mentioned in the paper. Section 4.1 mentions AutoGen version 0.9.2 in the case trajectory (Appendix D), but no systematic dependency listing is provided in the paper itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. While a GitHub link is provided and a dataset is mentioned, the paper does not include a 'Reproducing Results' section or describe specific commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7b shows 95% confidence intervals computed via bootstrap for the runtime-performance relationship. However, the main results in Table 1 do not include confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares multiple LLM backends (Table 1) and claims one outperforms others, but no statistical significance tests (p-values, t-tests, etc.) are reported. Comparisons are made purely by comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides baseline context for its claims. Section 6.2 reports '132 minutes per task' reduced to 'approximately two minutes, a reduction of over 98%.' Table 1 reports absolute scores across benchmarks with the baseline (no components) at 61.8 avg vs. full system at 88.5 avg."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset consists of 100 scenarios and the human evaluation uses 11 engineers, but no justification for these sample sizes is provided. No power analysis or discussion of whether these numbers are sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.1.1 states 'Each problem was evaluated over ten independent trials' but Table 1 reports only single point estimates per benchmark per model without standard deviation or any spread measure. Figure 7b shows bootstrap CI for runtime but not for the main performance table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares MASSE performance across multiple LLM backends (GPT-3.5-turbo, GPT-4o, Claude 3.5 Sonnet, o4-mini) in Table 1, and the ablation study (Table 2) compares against a no-component baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include contemporary models: GPT-4o (2024), Claude 3.5 Sonnet (2024), and o4-mini (2025). GPT-3.5-turbo is included as a weaker comparison point. These are current frontier models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents an ablation study varying agent memory (M) and JSON format (J) components across four benchmarks, showing the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation benchmarks are used: SAAB, SDAB, LAB, and MASEB, each measuring different aspects of the system's performance (structural analysis, design, loading, and holistic performance)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 reports a comparative study with 11 experienced structural engineers completing a standardized racking system design task, comparing conventional workflow (132 min avg) against MASSE (approx. 2 min)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no mention of splitting the 100 scenarios into training/development and test sets. The paper does not describe using a held-out portion, and since GPT-5 is used as a judge, it is unclear if any tuning was done on the same data used for final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-benchmark breakdowns (SAAB, SDAB, LAB, MASEB) for each model. Figure 2 provides a per-failure-mode breakdown for single-agent failures. The ablation study also shows per-benchmark results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 categorizes failure modes of the single-agent system into Formatting (20%), Dependency (50%), and Logic (30%). Appendix A provides further details on single- vs. two-agent failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the single-agent system failed in all 10 trials (Figure 2). GPT-3.5-turbo underperformance is explicitly discussed (67.7 MASEB). The ablation shows that removing components hurts performance substantially."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims MASSE 'can reduce expert workload from approximately two hours to mere minutes' which is supported by Section 6.2 (132 min to ~2 min). The claim of 'enhancing both reliability and accuracy' is supported by Table 1 benchmark scores. The 'first Multi-Agent System for Structural Engineering' claim appears consistent with the related work review."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about memory and JSON format improving performance (Table 2 ablation). The ablation study uses controlled single-variable manipulation (adding M, adding J, adding both) which is adequate for the causal claims about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Automating Structural Engineering Workflows' broadly, but the evaluation is limited to a single task type: racking system design in British Columbia. While Section 5.1 acknowledges this is 'a representative task,' the paper's discussion (Section 7) and conclusion extrapolate to 'other domains where tasks are verbalizable, procedural, and tool-mediated' without bounding these claims to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, the 98% time reduction could partly be due to the specific task being well-suited to automation rather than the multi-agent architecture. The GPT-5 judge evaluation could have systematic biases. No threats-to-validity or confound discussion is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 mentions 'GPT-4o', 'Claude 3.5 Sonnet', 'o4-mini', and 'GPT-3.5-turbo' without specific API version IDs or snapshot dates. 'GPT-4o' and 'Claude 3.5 Sonnet' are marketing names, not specific versions. No snapshot dates are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B (Figures 8-10) provides the complete system instruction prompts for all agents: Loading Analyst, Seismic Analyst, Dynamic Analyst, Structural Analyst, Design Engineer, Model Engineer, Verification Engineer, Project Manager, and Safety Manager. These are the actual prompt texts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'Temperatures are set to 0 for GPT-4o, Claude 3.5 Sonnet, and GPT-3.5-turbo, and 1 for o4-mini.' Appendix D also notes 'Max tokens = 2000'. The embedding model text-embedding-3 is mentioned for RAG."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: Section 3 covers the three-team role design, Section 4 covers communication protocols and agent interactions, Figure 3 shows the overall framework, and Appendix C provides detailed component descriptions. The AutoGen framework is identified as the orchestration layer."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 explains the dataset construction: 'constructed a domain-specific dataset consisting of one hundred different levels of difficulty, each paired with validated ground-truth solutions.' It notes the dataset was 'reorganized but faithfully mirrors the original cases' due to privacy constraints."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The paper has Discussion (Section 7) and Ethics Statement sections but neither serves as a limitations discussion. The Discussion covers Transparency, Safety, and Real-World Impact as positive framing."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address limitations of the GPT-5 judge, potential biases in the evaluation, the restricted domain scope, or threats from using a single task type."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The Ethics Statement says the system is 'for academic research and educational use' only, but this is a liability disclaimer, not a scope boundary about the generalizability of findings. The conclusion suggests 'future work will emphasize deployment in consulting environments' but does not explicitly bound current claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the dataset is said to be released, the paper does not mention releasing the raw experimental logs, traces, or individual trial results. The released data appears to be the problem scenarios, not the raw evaluation outputs from the 10 trials per problem."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes the data collection: a domain-specific dataset of 100 scenarios derived from real-world racking system projects in British Columbia, reorganized for privacy. Each sample contains natural language descriptions, intermediate reasoning steps, and ground-truth solutions."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '11 experienced structural engineers' in Section 6.2 but provides no details on how they were recruited, their specific experience levels, or whether they had any prior relationship with the authors. This could introduce selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides a detailed step-by-step trace of the full analysis pipeline from problem input through all 10 steps to final safety assessment. Appendix E details the simulation setup and evaluation metrics with rubrics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgement section states: 'This work was supported by Natural Sciences and Engineering Research Council of Canada (NSERC) through the Alliance Grant [ALLRP 581074-22].'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: University of Alberta and Duke University. The paper evaluates third-party LLMs (OpenAI, Anthropic models), not products from the authors' own institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is NSERC, a Canadian government funding agency that has no commercial stake in the outcome of LLM-based structural engineering automation research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o, Claude 3.5 Sonnet, GPT-3.5-turbo, and o4-mini on its custom benchmarks but does not state the training data cutoff dates for any of these models. While the dataset is custom, the models could have seen similar structural engineering problems."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the models may have been exposed to similar structural engineering problems or racking system designs during training. The dataset is derived from real-world projects, which could partially overlap with public engineering data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks (SAAB, SDAB, LAB, MASEB) are novel and custom, which partially mitigates contamination risk. However, the paper does not discuss contamination at all, nor does it address whether the underlying engineering knowledge tested could be in the training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.2 describes a study with 11 engineers but there is no mention of pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The Ethics Statement discusses data confidentiality for the human evaluation but does not mention IRB or ethics board approval."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.2 describes participants only as '11 experienced structural engineers.' No demographics are reported: years of experience, education level, geographic distribution, or other characteristics."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for the 11 engineer participants. It is unclear what 'experienced' means or how they were selected."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not a randomized experiment — all engineers completed the same task using the conventional workflow. There is no treatment/control comparison among human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not a blinded study — engineers were asked to complete a specific task manually. There was no comparison condition where blinding would be relevant."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information is provided about whether any engineers started but did not complete the study, or whether 11 was the initial or final number of participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7a provides a cost analysis showing token consumption (log10 scale) and runtime per method. Section 6.1.2 discusses the cost-performance tradeoff across models. However, actual dollar costs are not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While token consumption and runtime are visualized in Figure 7a, no total computational budget is stated (e.g., total API spend, total number of API calls across all experiments). The paper does not quantify the total cost of running 100 scenarios × 10 trials × 4 models."
       }
     }
   },
   "claims": [
     {
       "claim": "MASSE can reduce expert workload from approximately two hours to mere minutes (over 98% reduction).",
       "evidence": "Section 6.2: 11 engineers averaged 132 minutes per task; MASSE with GPT-4o completed in approximately 2 minutes.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning model o4-mini provides the most stable and generalizable performance across heterogeneous structural engineering tasks.",
       "evidence": "Table 1: o4-mini achieves highest scores on SAAB (96.6), SDAB (91.4), MASEB (94.7), and average (94.1), outperforming all language models.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-agent systems are necessary for long-horizon structural engineering tasks; single-agent fails 100% of the time.",
       "evidence": "Figure 2 and Appendix A: Single-agent (GPT-4o) failed in all 10 trials with errors distributed across formatting (20%), dependency (50%), and logic (30%). Two-agent system succeeded in every trial.",
       "supported": "moderate"
     },
     {
       "claim": "Agent memory and structured JSON I/O are crucial components, with their combination yielding the best results.",
       "evidence": "Table 2: Baseline (no M, no J) scores 61.8 avg; +M alone scores 66.5; +J alone scores 64.5; +M,J scores 88.5.",
       "supported": "strong"
     },
     {
       "claim": "MASSE is the first multi-agent system for structural engineering.",
       "evidence": "Introduction and abstract. The related work section (2.1) reviews prior LLM applications in civil engineering but identifies no prior multi-agent system for full structural engineering workflows.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "MASSE, a multi-agent LLM system for structural engineering, coordinates Analyst, Engineer, and Management agent teams to automate racking system design. Using custom benchmarks (SAAB, SDAB, LAB, MASEB) on 100 real-world-derived scenarios, the reasoning model o4-mini achieved the highest overall score (94.1 avg), while Claude 3.5 Sonnet and GPT-4o also performed well. A human evaluation with 11 structural engineers showed MASSE reduced task completion time from 132 minutes to approximately 2 minutes (98% reduction). Ablation studies demonstrated that both structured memory and JSON-based communication are essential components.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated limitations or threats-to-validity discussion. The Discussion section (Section 7) frames everything positively (transparency, safety, real-world impact) without acknowledging weaknesses."
     },
     {
       "flag": "GPT-5 as automated judge without validation",
       "detail": "The evaluation uses GPT-5 as an LLM Judge (Section E.2.1) to score MASSE outputs against ground truth. There is no discussion of the judge's reliability, potential biases, or correlation with human expert judgment. LLM judges can have systematic biases."
     },
     {
       "flag": "Single task domain generalized broadly",
       "detail": "All evaluation is on racking system design in British Columbia, but the paper title, abstract, and discussion generalize to 'structural engineering workflows' broadly and even to 'knowledge-intensive sectors such as architecture, finance, and healthcare.'"
     },
     {
       "flag": "No variance reported for main results",
       "detail": "Despite running 10 trials per problem, Table 1 reports only point estimates without standard deviations or confidence intervals. The reader cannot assess result stability."
     },
     {
       "flag": "Minimal human study methodology",
       "detail": "The human evaluation (11 engineers) lacks demographics, recruitment methods, inclusion criteria, IRB approval, and attrition reporting. The comparison is between manual task completion time and automated system time, which conflates many factors."
     },
     {
       "flag": "Self-citation cluster",
       "detail": "Several references (e.g., [17], [18], [31], [37], [41]-[43], [60]-[64]) appear to be works by co-authors (Liang, Zhou, Song, Shen) in unrelated areas (model compression, diffusion models, circuit complexity). These bulk up the reference list without contributing to the paper's technical narrative."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Core multi-agent orchestration framework used as the backbone of MASSE."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Systematic analysis of failure modes in multi-agent LLM systems, directly relevant to understanding agentic system reliability."
     },
     {
       "title": "AI Agents That Matter",
       "authors": ["Sayash Kapoor", "Benedikt Stroebl", "Zachary S Siegel"],
       "year": 2024,
       "arxiv_id": "2407.01502",
       "relevance": "Highlights the need for rigorous benchmarking practices in LLM agent evaluation."
     },
     {
       "title": "Magentic-One: A Generalist Multi-Agent System for Solving Complex Tasks",
       "authors": ["Adam Fourney", "Gagan Bansal", "Hussein Mozannar"],
       "year": 2024,
       "arxiv_id": "2411.04468",
       "relevance": "General-purpose multi-agent system for complex task-solving, providing context for domain-specific multi-agent architectures."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2023,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent LLM framework for software engineering, a closely related domain to structural engineering automation."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong", "Xiawu Zheng", "Jonathan Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework coordinating agents for software tasks, relevant to understanding role-based agent architectures."
     },
     {
       "title": "HyperAgent: Generalist Software Engineering Agents to Solve Coding Tasks at Scale",
       "authors": ["Huy Nhat Phan", "Tien N Nguyen"],
       "year": 2024,
       "arxiv_id": "2409.16299",
       "relevance": "Generalist software engineering agent evaluated at scale, relevant to benchmarking agentic programming systems."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "relevance": "Foundational agent reasoning framework that MASSE builds upon for structured workflows."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "relevance": "Structured reasoning framework for LLMs, foundational to the agent reasoning approaches used in MASSE."
     },
     {
       "title": "The Illusion of Diminishing Returns: Measuring Long Horizon Execution in LLMs",
       "authors": ["Akshit Sinha", "Arvindh Arun", "Shashwat Goel"],
       "year": 2025,
       "arxiv_id": "2509.09677",
       "relevance": "Directly relevant to understanding why single-agent systems fail on long-horizon tasks, motivating the multi-agent approach."
     },
     {
       "title": "Taxonomy of Failure Modes in Agentic AI Systems: Whitepaper",
       "authors": ["Microsoft Corporation"],
       "year": 2025,
       "relevance": "Systematic taxonomy of failure modes in agentic AI systems, relevant to understanding and categorizing agent failures."
     },
     {
       "title": "Gödel Agent: A Self-Referential Agent Framework for Recursive Self-Improvement",
       "authors": ["Xunjian Yin", "Xinyi Wang", "Liangming Pan"],
       "year": 2024,
       "arxiv_id": "2410.04444",
       "relevance": "Self-evolving agent framework relevant to the survey's coverage of advanced agentic architectures."
     }
   ]
 }