ai-research-survey

scan.json (20106B)

---

 {
   "paper": {
     "title": "Advancing engineering research through context-aware and knowledge graph–based retrieval-augmented generation",
     "authors": ["Soham Ghosh", "Gaurav Mittal"],
     "year": 2025,
     "venue": "Frontiers in Artificial Intelligence",
     "doi": "10.3389/frai.2025.1697169"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in the Data Availability section: https://github.com/sghosh27/Low-Code-RAG-LLM-Framework-for-Context-Aware-Querying-in-Electrical-Standards-Design-and-Research along with Python code for document parsing and a scoring notebook."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The n8n workflow templates and Python notebooks are released via the GitHub repository. The evaluation queries and ground truth answers are published in Tables 2 and 5 of the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is mentioned. The paper names tools (LM Studio, AnythingLLM, n8n, InfraNodus) but does not specify library versions or dependencies needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While n8n workflow JSON files are released, there are no step-by-step reproduction instructions describing how to set up the full pipeline, configure the tools, or replicate the experiments end-to-end."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports median and interquartile range (IQR) for contextual vs traditional RAG: 0.83 [0.79, 0.89] and 0.62 [0.26, 0.77] respectively."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "A one-sided paired Wilcoxon Signed test was conducted to compare contextual RAG vs traditional RAG, reporting p < 0.0001 at significance level 0.05."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Median scores with baselines are reported: contextual RAG median 0.83 vs traditional RAG median 0.62, providing sufficient context to understand the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the number of evaluation queries used. The sample sizes (6 queries in Table 2, 3 in Table 5, 2 in Table 6) are small with no discussion of whether this is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "IQR is reported for contextual and traditional RAG scores: 0.83 [0.79, 0.89] and 0.62 [0.26, 0.77]."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Traditional RAG is used as a baseline against contextual RAG (Table 5). For reranking experiments, a no-reranking baseline is included alongside bge-reranker-base and Cohere V3.5 (Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include current tools: Cohere Rerank V3.5, bge-reranker-base, OpenAI embeddings, Voyage embeddings, Google-PaLM embeddings, which are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted to isolate which specific components of the contextual RAG pipeline contribute most to the improvement. The comparisons are between whole systems."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Hit rate and MRR are used for reranking evaluation (Table 3). GEval correctness scoring is used for response quality evaluation (Tables 2 and 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is conducted. Evaluation relies entirely on automated GEval scoring against ground truth answers. Practitioner engineers helped design queries and ground truth but did not evaluate outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no mention of held-out test sets. The same queries appear to be used for both development and evaluation, with no separation described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Individual query-level scores are reported in Tables 2 and 5, showing per-query performance for both traditional and contextual RAG."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses failures with numerical table retrieval, multi-condition exceptions, and chunking limitations. Q5 and Q6 in Table 2 show low scores (0.26, 0.20) with explanations of why the system failed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that traditional RAG failed on table lookups (Q6 scored 0.20), knowledge graphs are poor for numerical tabular data, and both RAG approaches still hallucinate on complex multi-layered prompts."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the frameworks improve relevance and promote deeper semantic understanding, which are supported by the experimental results in Tables 2, 3, and 5 showing improvements from reranking, contextual RAG, and knowledge graph RAG."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims contextual RAG 'improves' over traditional RAG, which is a causal claim. While a Wilcoxon test is used, the sample of queries is very small (3 paired comparisons in Table 5) and not randomly sampled, making the causal design weak."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Advancing engineering research' broadly, but results are tested only on the 2017 NEC codebook and a small set of electrical engineering queries. No bounding of generalization to this specific domain is stated in the abstract or title."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed improvements. For example, the improvement from contextual RAG could be due to larger chunk sizes rather than the contextual augmentation itself, but this is not considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Llama 4 Scout 17B 16E' for contextual RAG and lists models like 'Cohere Rerank v3.5' and 'bge-reranker-base', but does not specify exact versions or snapshot dates for the LLMs used in the main experiments (Tables 2 and 5). The specific model used for the traditional RAG experiments is not clearly stated."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5B shows the actual prompt template used for contextual RAG. The evaluation prompts are provided verbatim in Tables 2, 5, and 6. The InfraNodus system message is also quoted."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that temperature and top-K can be tuned in LM Studio but does not report the actual values used. Chunk size of 2,000 with overlap of 20 is stated for Table 5, but other hyperparameters (temperature, top-K, top-p) are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The n8n workflow architecture is described in detail with workflow diagrams (Figures 3, 4, 5, 7) showing document ingestion, embedding, vector storage, retrieval, reranking, and LLM generation steps."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes converting PDFs to Markdown, chunking strategies (chunk size 2,000, overlap 20), and the embedding process. The contextual RAG preprocessing with parent context augmentation is also described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Summary and future work) discusses limitations including higher API costs for knowledge graph generation, limitations with numerical tabular data, and remaining hallucination issues. Limitations are also discussed throughout Sections 4 and 5."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity section exists. The limitations discussed are about the RAG systems' functional shortcomings, not about threats to the validity of the study's findings (e.g., small sample size, selection of queries, lack of generalizability)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to the NEC domain or acknowledge that results may not transfer to other engineering standards or document types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw evaluation scores and individual query results are shown in tables, but the underlying NEC document used (proprietary NFPA content) is not available, and the full set of scoring outputs is not released as raw data files."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes that queries were developed collaboratively with practicing electrical engineering professionals, designed to cover NEC chapters, with ground truth developed by those practitioners."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Practitioner engineers helped design queries but were not study subjects. The data source is a standard engineering codebook (NEC 2017)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from document upload through parsing, embedding, vector storage, retrieval, and LLM generation is documented across Sections 3-6 with workflow diagrams."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Funding section states: 'The author(s) declare that no financial support was received for the research and/or publication of this article.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with Black & Veatch are disclosed in the header and in the Conflict of Interest section: 'SG and GM were employed at Black & Veatch.'"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "The paper declares no financial support was received, making this criterion not applicable (unfunded work)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The Conflict of Interest section discloses employment at Black & Veatch. The Author Disclaimer states the study was conducted independently without institutional or agency sponsorship."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LLMs to answer questions about the NEC but does not state the training data cutoff for any model used. The LLMs may have seen NEC content during training, which would inflate RAG performance measurements."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the NEC 2017 content or similar engineering standards appeared in the LLMs' training data, which could confound the evaluation of RAG effectiveness."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The NEC 2017 is a widely available document that may have been in the training data of the models used. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses cost qualitatively (e.g., knowledge graph RAG uses 5-10x more tokens, Table 4 lists 'Embedding cost' as 'Low' vs 'Higher') but does not report actual inference costs, API spend, or per-query costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is reported despite the paper using multiple commercial APIs (OpenAI, Cohere, Voyage)."
       }
     }
   },
   "claims": [
     {
       "claim": "Contextual RAG significantly outperforms traditional RAG on NEC-based engineering queries",
       "evidence": "Wilcoxon Signed test p < 0.0001; median scores 0.83 vs 0.62 (Section 5.2, Table 5)",
       "supported": "moderate"
     },
     {
       "claim": "OpenAI + Cohere rerank combination achieves the highest retrieval performance",
       "evidence": "Table 3 shows OpenAI + Cohere V3.5 achieving hit rate 0.932 and MRR 0.877, highest across all configurations",
       "supported": "moderate"
     },
     {
       "claim": "Knowledge graph-based RAG can traverse multiple linked knowledge graphs to answer cross-domain queries",
       "evidence": "Table 6 shows two qualitative examples where the AI agent consulted multiple 'brains' to compose answers",
       "supported": "weak"
     },
     {
       "claim": "Traditional RAG struggles with numerical table retrieval and multi-condition exceptions in engineering codes",
       "evidence": "Table 2 Q5 (score 0.26) and Q6 (score 0.20) demonstrate failures; qualitative analysis in Section 4",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper presents rapidly deployable RAG-LLM pipelines using n8n for engineering document querying. Contextual RAG significantly outperforms traditional RAG on NEC-based queries (median 0.83 vs 0.62, p < 0.0001). Reranking with Cohere V3.5 consistently improves retrieval metrics across embedding models. Knowledge graph-based RAG enables cross-domain exploratory queries but at 5-10x higher API cost, and both traditional and contextual RAG still hallucinate on complex multi-layered prompts.",
   "red_flags": [
     {
       "flag": "Very small evaluation sample",
       "detail": "Only 6 queries for traditional RAG (Table 2), 3 paired comparisons for contextual vs traditional RAG (Table 5), and 2 queries for knowledge graph RAG (Table 6). Statistical claims from such small samples are fragile."
     },
     {
       "flag": "No contamination analysis",
       "detail": "The NEC 2017 is a widely available document. LLMs may have seen it during training, meaning the RAG system's apparent accuracy could be partly due to the model's pre-existing knowledge rather than retrieval quality."
     },
     {
       "flag": "Automated evaluation only",
       "detail": "All evaluation uses DeepEval's GEval metric, which itself relies on an LLM to score outputs. No human expert evaluated whether the RAG system's answers were correct for engineering decision-making."
     },
     {
       "flag": "Unspecified model versions",
       "detail": "The exact LLM models and versions used for the main experiments are not clearly stated, making reproduction difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "HaluEval: a large-scale hallucination evaluation benchmark",
       "authors": ["J. Li", "X. Cheng", "W. X. Zhao", "J.Y. Nie", "J.R. Wen"],
       "year": 2023,
       "relevance": "Benchmark for evaluating LLM hallucination, relevant to methodology quality in LLM evaluation."
     },
     {
       "title": "Hallucination detection: robustly discerning reliable answers in large language models",
       "authors": ["Y. Chen", "Q. Fu", "Y. Yuan", "Z. Wen", "G. Fan", "D. Liu"],
       "year": 2023,
       "relevance": "Methods for detecting LLM hallucinations, relevant to LLM reliability and safety research."
     },
     {
       "title": "Retrieval Augmented Generation using engineering design knowledge",
       "authors": ["L. Siddharth", "J. Luo"],
       "year": 2024,
       "doi": "10.1016/j.knosys.2024.112410",
       "relevance": "RAG applied to engineering domains, directly relevant to evaluating RAG methodology quality."
     },
     {
       "title": "Enhancing engineering education through LLM-driven adaptive quiz generation: a RAG-based approach",
       "authors": ["S. Gopi", "D. Sreekanth", "N. Dehboz"],
       "year": 2024,
       "doi": "10.1109/FIE61694.2024.10893146",
       "relevance": "RAG application in engineering education, relevant to LLM application methodology."
     },
     {
       "title": "Large language models are highly vulnerable to adversarial hallucination attacks in clinical decision support: a multi-model assurance analysis",
       "authors": ["M. Omar", "V. Sorin", "J. D. Collins"],
       "year": 2025,
       "doi": "10.1101/2025.03.18.25324184",
       "relevance": "LLM vulnerability to adversarial attacks causing hallucinations, relevant to LLM safety."
     },
     {
       "title": "Enhancing uncertainty-based hallucination detection with stronger focus",
       "authors": ["T. Zhang", "L. Qiu", "Q. Guo"],
       "year": 2023,
       "relevance": "Hallucination detection methodology for LLMs, relevant to LLM reliability research."
     }
   ]
 }