ai-research-survey

scan.json (28367B)

---

 {
   "paper": {
     "title": "Generative AI in the Construction Industry: A State-of-the-art Analysis",
     "authors": [
       "Ridwan Taiwo",
       "Idris Temitope Bello",
       "Sulemana Fatoama Abdulai",
       "Abdul-Mugis Yussif",
       "Babatunde Abiodun Salami",
       "Abdullahi Saka",
       "Tarek Zayed"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2402.09939",
     "doi": "10.48550/arXiv.2402.09939"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis", "case-study", "qualitative"],
   "key_findings": "A systematic review found only 6 peer-reviewed papers on generative AI in construction, indicating very early-stage adoption. An expert Delphi panel identified numerous opportunities across text, image, and video modalities for pre-construction, construction, and post-construction phases. A case study demonstrated that retrieval-augmented generation (RAG) improved GPT-4's contract document querying by 5.2%, 9.4%, and 4.8% in quality, relevance, and reproducibility respectively, though the RAG system's answer rate dropped from 100% to 90%.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or archive is mentioned anywhere in the paper. The RAG system built with LangChain and Streamlit is described but no code is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The contract document used in the case study was obtained from a consultancy firm and is not released. The expert survey responses are not shared. No datasets are made publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency lists are provided. The paper mentions LangChain, Streamlit, OpenAI API, and Cassandra but does not specify versions or provide setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are included. The RAG pipeline is described at a high level (Section 6.1) but lacks the specificity needed to reproduce the system."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Table 21 presents averaged ratings from 3 experts with no uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The claim that RAG improves GPT-4 is based solely on comparing average ratings (e.g., 3.87 vs 4.13 for quality) without any test of whether the difference is statistically meaningful."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both baseline and improved scores with percentage improvements: quality from 3.87 (77.4%) to 4.13 (82.6%), relevance from 4.01 (80.2%) to 4.48 (89.6%), reproducibility from 4.53 (90.6%) to 4.77 (95.4%), providing sufficient context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for using 20 evaluation questions, 3 expert raters, or 11 Delphi panelists. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or inter-rater agreement metrics are reported. Individual expert ratings are not disaggregated beyond the averages in Table 21."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The case study compares GPT-4 (baseline) against GPT-4+RAG on the same 20 questions, providing a direct baseline comparison (Table 21)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 was state-of-the-art at the time of the study (2024). Using GPT-4 as the baseline for evaluating RAG augmentation is appropriate."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The system has only one added component (RAG) on top of the base GPT-4 model, making a multi-component ablation study inapplicable."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation metrics are used: answer rate, quality (1-5), relevance (1-5), and reproducibility (1-5), as described in Section 6.2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Three domain experts evaluated the model outputs on the four metrics. Section 6.2 states '3 experts from the original panel' assessed responses to each question."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No dev/test separation is described. The 20 expert-validated questions were used directly for evaluation without indication of a separate development or tuning set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 21 provides per-question scores for all 20 questions across all four metrics for both GPT-4 and GPT-4+RAG, enabling detailed inspection of individual question performance."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses failure cases: the RAG model failed to answer 2 of 20 questions. Figure 11 provides a side-by-side comparison showing GPT-4 hallucinating details about GCC Clause 44 while RAG correctly extracted the answer."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The RAG system's lower answer rate (90% vs 100% for baseline GPT-4) is reported as a negative result. Section 6.3 acknowledges 'the model struggled to retrieve relevant passages for two of the questions.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's specific claims are supported: the 5.2%, 9.4%, and 4.8% improvements are verified in Table 21; the review of opportunities and challenges is presented in Section 4; the framework is in Section 5; the case study is in Section 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claim is that RAG improves GPT-4's performance. This is tested via controlled single-variable manipulation: the same GPT-4 model with and without RAG, evaluated on the same 20 questions by the same raters. This design is adequate for the causal claim, despite the small sample."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames itself as 'a comprehensive analysis and practical framework' and 'a state-of-the-art analysis' of generative AI in construction. However, the evidence base is thin: only 6 papers found in the systematic review, 11 experts surveyed, and 1 case study on a single contract document with a single model. The broad framing significantly exceeds the narrow evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the RAG improvement. Possible confounds such as evaluator bias (experts knew which system was which), question selection effects, or small-sample variability are not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures subjective 5-point expert ratings of quality, relevance, and reproducibility and frames these as evidence of improved 'information retrieval and knowledge discovery' (Section 6). The gap between subjective ratings by 3 experts and actual information extraction quality is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'GPT-4' throughout without specifying a version or snapshot date (e.g., gpt-4-0613). No API version or model checkpoint is identified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided for the RAG system or baseline GPT-4 queries. The 20 evaluation questions are referenced but not fully listed in the paper. The system prompt used with the LLM is not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters are reported (temperature, top-p, max tokens). The RAG system's chunk size is described as '3-5 sentences' but the top-k retrieval parameter, embedding model specifics, and similarity threshold are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The system is a fixed RAG pipeline (retrieve-augment-generate) without agentic scaffolding such as autonomous tool use, retry logic, feedback mechanisms, or multi-step reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Phase 1 documents the literature search pipeline with databases, search string, screening counts (79→10→4 original + 2 review), and snowball expansion to 6 papers. The case study describes document chunking into 3-5 sentence segments with embedding and vector storage steps (Section 6.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 ('Model limitation') is a dedicated subsection discussing case study limitations. Section 7 (Conclusion) contains a substantive paragraph on overall study limitations covering database scope, panel size, and model choice."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: the RAG model failed on 2 questions due to chunking strategy limitations (Section 6.3); the literature review was confined to 3 databases despite snowball searching; the Delphi panel was restricted in size; only a single LLM and embedding technique were used due to API costs (Section 7)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states specific boundaries: 'confined to three databases - Scopus, Web of Science, and ScienceDirect'; 'only a single base large language model and embedding technique were utilized due to API access costs'; model 'trained on just a single contract document and may fail to transfer to new projects' (Section 6.3)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The expert survey responses, contract document, model outputs, and individual expert ratings are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Phase 1 describes literature search with specific databases and search strings. Phase 2 describes expert recruitment: 15 experts invited, 11 accepted (73% response rate), with backgrounds spanning AI research and construction practice. Phase 4 describes case study data collection from a consultancy firm."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the expert panel composition ('university professors in AI and construction engineering, technology directors from major construction firms, and founders of AI startups targeting the AEC industry') and response rate, but does not explain how the 15 experts were identified or contacted, introducing potential selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The literature search pipeline has counts (79→10→6), but the expert Delphi analysis pipeline is underdocumented: the thematic analysis process, coding scheme, inter-coder reliability, and aggregation from individual responses to categorized opportunities/challenges are not detailed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section states: 'This research is supported by the Department of Building and Real Estate, The Hong Kong Polytechnic University, and the Centre for Advances in Reliability and Safety (CAiRS).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Hong Kong Polytechnic University, CAiRS, Cardiff Metropolitan University, and Leeds Beckett University. The paper does not evaluate a product from any of these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from a university department and research center with no apparent financial interest in whether generative AI succeeds or fails in the construction industry."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The case study tests RAG-augmented information extraction from a proprietary contract document, not model knowledge on a public benchmark. Contamination of the specific contract document is not a meaningful concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation queries are custom-created for a specific proprietary contract document, not a public benchmark that could be in GPT-4's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No standard benchmark is used. The evaluation is on 20 custom questions about a proprietary contract document."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for either the Delphi expert study or the case study evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite involving 11 expert participants in a Delphi survey and 3 expert evaluators."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only broad categories are given for the expert panel ('university professors in AI and construction engineering, technology directors from major construction firms, and founders of AI startups'). No individual demographics such as years of experience, specific expertise levels, or geographic distribution are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for expert selection are stated. The paper says '15 experts with backgrounds spanning AI research and construction industry practice were identified' without explaining the selection criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The study involves a Delphi survey and expert evaluation, not an experimental design with randomized condition assignment."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described for the case study evaluation. The 3 expert raters evaluated both GPT-4 and GPT-4+RAG outputs, but there is no indication they were blinded to which system produced which responses, introducing potential bias."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Attrition is reported: '15 experts with backgrounds spanning AI research and construction industry practice were identified. Invitations were sent to participate in the study, with 11 experts accepting for a 73% response rate.'"
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or inference times are reported for the GPT-4 or RAG system queries. Section 7 mentions 'API access costs' as a limiting factor but does not quantify them."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The hardware used, API spend, or processing time for the RAG system are not reported."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "Phase 1 describes a structured search strategy with specified databases (Scopus, Web of Science, ScienceDirect), an explicit reproducible search string, screening with counts (79→10→6), and snowball searching. However, no PRISMA flow diagram or protocol registration is provided."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "No quality assessment of the 6 reviewed papers is conducted. Table 10 summarizes the papers' objectives, methods, and contributions but does not evaluate their methodological quality, risk of bias, or rigor."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The paper does not consider whether the 6 identified studies might skew positive or whether negative results on generative AI in construction are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "Generative AI adoption in the construction industry is still in very early stages, with only 6 peer-reviewed articles found exploring uses of LLMs in construction.",
       "evidence": "Systematic literature search across Scopus, Web of Science, and ScienceDirect with snowball searching identified only 6 peer-reviewed papers (Section 4.1, Table 10).",
       "supported": "moderate"
     },
     {
       "claim": "RAG improves the baseline GPT-4 LLM by 5.2%, 9.4%, and 4.8% in terms of quality, relevance, and reproducibility for querying contract documents.",
       "evidence": "Table 21 shows GPT-4 averaged 3.87/4.01/4.53 vs GPT-4+RAG at 4.13/4.48/4.77 on quality/relevance/reproducibility scales rated by 3 experts on 20 questions (Section 6.2).",
       "supported": "weak"
     },
     {
       "claim": "GPT-4 without RAG hallucinates plausible-sounding but incorrect details from contract documents, while RAG grounds outputs in actual document content.",
       "evidence": "Figure 11 shows a side-by-side comparison where GPT-4 fabricates details about GCC Clause 44 while GPT-4+RAG correctly extracts the price adjustment formula from the contract (Section 6.2).",
       "supported": "moderate"
     },
     {
       "claim": "RAG-augmented GPT-4 has a lower answer rate (90%) compared to baseline GPT-4 (100%) because it sometimes cannot link queries to relevant document passages.",
       "evidence": "Table 21 shows RAG answered 18/20 questions vs 20/20 for baseline. Section 6.3 explains 'the chunking strategies and semantic search techniques used were unable to adequately link some complex questions to supporting evidence.'",
       "supported": "moderate"
     },
     {
       "claim": "At least seven potential opportunities for generative AI exist for each major input-output data type (text-text, text-image, image-text, etc.) across construction project phases.",
       "evidence": "Tables 12-20 list 7-11 potential opportunities for each of the 9 input-output types, identified through expert Delphi discussion (Sections 4.2.1-4.2.9).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Extremely thin evidence base for a 'comprehensive' survey",
       "detail": "The systematic literature review found only 6 peer-reviewed papers, yet the paper frames itself as 'a comprehensive analysis' and 'state-of-the-art analysis' of generative AI in construction. The bulk of the opportunities and challenges are derived from 11 expert opinions rather than empirical evidence."
     },
     {
       "flag": "No statistical tests for improvement claims",
       "detail": "The claim that RAG improves GPT-4 by 5.2-9.4% is based on comparing averages from 3 expert raters on 20 questions with no significance tests, confidence intervals, or inter-rater reliability metrics. The differences could easily be noise at this sample size."
     },
     {
       "flag": "Tiny evaluation sample",
       "detail": "The case study uses only 20 questions, 3 raters, and 1 contract document. This is insufficient to support generalizable claims about RAG's effectiveness for construction information retrieval."
     },
     {
       "flag": "Survey without quality assessment of sources",
       "detail": "The 6 reviewed papers are summarized but never assessed for methodological quality. The survey treats all papers equally regardless of rigor, laundering the signal-to-noise ratio of its sources."
     },
     {
       "flag": "Expert panel recruitment unclear",
       "detail": "The 15 invited experts were 'identified' through an undescribed process. The selection method could introduce significant bias toward experts favorable to AI adoption in construction. No demographics, experience levels, or selection criteria are provided."
     },
     {
       "flag": "No evaluator blinding",
       "detail": "Expert raters evaluated both GPT-4 and GPT-4+RAG outputs without described blinding. Knowledge of which system produced which output could bias ratings, especially given the paper's framing of RAG as an improvement."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational Codex evaluation paper establishing LLM code generation capabilities and benchmarks."
     },
     {
       "title": "GPT models in construction industry: Opportunities, limitations, and a use case validation",
       "authors": ["A. Saka", "R. Taiwo", "N. Saka", "B.A. Salami"],
       "year": 2024,
       "doi": "10.1016/j.dibe.2023.100300",
       "relevance": "Directly evaluates GPT models for construction industry tasks including material selection case study."
     },
     {
       "title": "Harnessing the Power of LLMs in Practice: A Survey on ChatGPT and Beyond",
       "authors": ["J. Yang", "H. Jin", "R. Tang"],
       "year": 2023,
       "relevance": "Comprehensive survey of practical LLM applications and capabilities relevant to the broader LLM evaluation landscape."
     },
     {
       "title": "Active Retrieval Augmented Generation",
       "authors": ["Z. Jiang", "F. Xu", "L. Gao"],
       "year": 2023,
       "doi": "10.18653/v1/2023.emnlp-main.495",
       "relevance": "RAG methodology paper relevant to understanding retrieval-augmented approaches for LLM applications."
     },
     {
       "title": "Dynamic prompt-based virtual assistant framework for BIM information search",
       "authors": ["J. Zheng", "M. Fischer"],
       "year": 2023,
       "doi": "10.1016/j.autcon.2023.105067",
       "relevance": "Integrates GPT with BIM for construction information retrieval, demonstrating LLM application in a domain-specific context."
     },
     {
       "title": "Robot-Enabled Construction Assembly with Automated Sequence Planning Based on ChatGPT: RoboGPT",
       "authors": ["H. You", "Y. Ye", "T. Zhou"],
       "year": 2023,
       "doi": "10.3390/buildings13071772",
       "relevance": "Uses ChatGPT for automated planning in robotic assembly, demonstrating agentic LLM use in physical construction tasks."
     },
     {
       "title": "Leveraging ChatGPT to Aid Construction Hazard Recognition and Support Safety Education and Training",
       "authors": ["S.M.J. Uddin", "A. Albert", "A. Ovid"],
       "year": 2023,
       "doi": "10.3390/su15097121",
       "relevance": "Evaluates ChatGPT's effectiveness for safety-critical hazard recognition tasks with human participants."
     },
     {
       "title": "Automated detection of contractual risk clauses from construction specifications using BERT",
       "authors": ["S. Moon", "S. Chi", "S.B. Im"],
       "year": 2022,
       "doi": "10.1016/j.autcon.2022.104465",
       "relevance": "Applies BERT transformer model for automated document analysis in construction, relevant to LLM capability evaluation."
     },
     {
       "title": "Generative AI design for building structures",
       "authors": ["W. Liao", "X. Lu", "Y. Fei"],
       "year": 2024,
       "doi": "10.1016/j.autcon.2023.105187",
       "relevance": "Reviews generative AI for structural design, evaluating AI capability in a complex engineering domain."
     },
     {
       "title": "Opportunities and Challenges of Generative AI in Construction Industry: Focusing on Adoption of Text-Based Models",
       "authors": ["P. Ghimire", "K. Kim", "M. Acharya"],
       "year": 2024,
       "relevance": "Closely related survey on generative AI adoption challenges in construction, focusing on text-based model limitations."
     },
     {
       "title": "A survey of Generative AI Applications",
       "authors": ["R. Gozalo-Brizuela", "E.C. Garrido-Merchán"],
       "year": 2023,
       "arxiv_id": "2306.02781",
       "relevance": "Broad survey of generative AI applications across domains, providing context for domain-specific AI capability assessments."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Proposes a framework and demonstrates a RAG case study for construction contract querying, but no code or tool is released for practitioners to use."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected narrative that generative AI has potential in construction but adoption is early — no surprising or contrarian findings."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Discusses hallucination risks and safety concerns of using generative AI in safety-critical construction tasks, but does not demonstrate novel attacks or existential risks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict — the paper presents a balanced view of opportunities and challenges."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released despite describing a Streamlit-based interface."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses GPT-4 (OpenAI) as the base model, but the paper is from academic authors at Hong Kong Polytechnic University, not a famous AI lab."
     }
   }
 }