ai-research-survey

scan.json (20566B)

---

 {
   "paper": {
     "title": "CyberBOT: Towards Reliable Cybersecurity Education via Ontology-Grounded Retrieval Augmented Generation",
     "authors": ["Chengshuai Zhao", "Riccardo De Maria", "Tharindu Kumarage", "Kumar Satvik Chaudhary", "Garima Agrawal", "Yiwen Li", "Jongchan Park", "Yuli Deng", "Ying-Chih Chen", "Huan Liu"],
     "year": 2025,
     "venue": "Arizona State University"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows a 'Quick Web App Setup' section with step-by-step deployment instructions including cloning a repository. The paper references a footnote for CyberBOT and shows deployment commands."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses CyberQ (Agrawal et al., 2024b), described as 'an open-source dataset comprising approx 3,500 open-ended cybersecurity QA pairs.' The course-specific knowledge base is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Docker, A100 80GB GPU, Together AI API, Streamlit, SQLite, LangChain, and LlamaIndex, but no requirements.txt, Dockerfile contents, or detailed dependency versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Figure 4 shows a quick setup guide screenshot but the actual commands and steps are not readable in the paper text. No detailed reproduction instructions for the experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates (e.g., BERTScore 0.933) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims across question categories (ZS, FS, OD) but provides no statistical significance tests. The planned ANCOVA for the field study has not yet been conducted."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Results are presented as raw metric scores without baseline context or magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The quasi-experimental study uses 77 students (39 experimental, 38 control) but no power analysis or justification for this sample size is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for the computational evaluation results in Table 1."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baseline comparisons are included. Table 1 reports only CyberBOT's own metrics without comparing to any alternative system, vanilla LLM, or prior QA approach."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so contemporaneity cannot be assessed."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "CyberBOT has multiple components (intent interpreter, RAG retriever, ontology verifier) but no ablation study is conducted to measure their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: BERTScore, METEOR, ROUGE-1, ROUGE-2 (QA-based) and Faithfulness, Answer Relevancy, Context Precision, Context Recall, Context Entity Recall (RAG-based)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a planned field study with surveys and interviews but states 'a forthcoming field study will evaluate its pedagogical impact.' Results are not yet available. The system makes claims about educational trustworthiness that warrant human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No mention of train/test/validation splits for the CyberQ evaluation dataset. It is unclear whether any tuning was done on the same data used for reporting."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-category breakdowns across Zero-shot (ZS), Few-shot (FS), and Ontology-Driven (OD) question types."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that 'the Faithfulness score is slightly lower because the model will leverage its own knowledge to generate answers when there is no closely relevant material in the knowledge base.' The limitations section also discusses failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results or failed approaches are reported. Every metric shows positive performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims CyberBOT 'reduc[es] the risk of misleading or unsafe guidance' but no comparison to a system without ontology validation is provided to support this claim. The abstract also claims 'Computational evaluations highlight the potential capacity' which is vaguely supported by Table 1 but without baselines."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the ontology verifier 'reduces the risk of misleading or unsafe guidance' and that the framework 'benefits from related documents as references,' but provides no ablation or controlled comparison to justify these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly states 'the current deployment focuses on a single graduate-level course with a limited sample size; thus, findings may not generalize to diverse educational settings or other technical domains.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider whether high BERTScore/ROUGE might be due to the QA pairs being derived from similar source material rather than system quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Llama 3.3 70B' for the intent interpreter, RAG generator, and ontology verifier, and 'BAAI-Bge-Large-1.5' for embeddings (Figure 3 and Section 3.2)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides the full prompts for the intent interpreter (C.1), the LLM generation (C.2), and the ontology verifier (C.3), including the actual template text and few-shot examples."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM hyperparameters are reported. The chunk size of 512 tokens for the knowledge base is mentioned but inference parameters are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage pipeline (intent interpreter, RAG retriever with FAISS, ontology verifier) is described in detail in Sections 2.1-2.3 with workflow diagrams (Figures 1-2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions PDFs are 'preprocess[ed] into smaller, semantically meaningful chunks' with 512 tokens, but does not describe how course materials were cleaned, filtered, or transformed beyond this brief mention."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present, discussing four specific limitations including knowledge base coverage, limited deployment scope, ontology coverage gaps, and computational overhead."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'the current deployment focuses on a single graduate-level course with a limited sample size' and 'the ontology-based validation primarily checks compliance with known concepts and relationships, leaving truly novel or emergent cybersecurity issues outside its purview.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states findings 'may not generalize to diverse educational settings or other technical domains' and identifies that novel cybersecurity issues are outside the system's purview."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw evaluation data, model outputs, or interaction logs are made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The CyberQ dataset source is described (Section 4.2), and the quasi-experimental study design describes data collection via pre/post surveys, learning outcomes, and interviews (Section 4.3)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes recruiting 77 graduate students from CSE 546 at ASU, with Monte Carlo random assignment stratified by gender into experimental (39) and control (38) groups."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from CyberQ dataset to final metrics in Table 1 is not documented. It is unclear how the 3,500 QA pairs were processed, whether any were filtered, or how the evaluation was run."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed as Arizona State University (School of Computing and Mary Lou Fulton Teachers College)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Absence of disclosure is not absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama 3.3 70B but does not state its training data cutoff date. The CyberQ dataset could potentially overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CyberQ QA pairs or related cybersecurity content appeared in Llama 3.3's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CyberQ was published in 2024 and could be in Llama 3.3's training data. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the quasi-experimental study with 77 students."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite collecting data from human participants. The Ethical Considerations section discusses privacy and consent but does not mention IRB approval."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Beyond stating participants are 'computer science ASU graduate students' stratified by gender, no demographic details (age, experience level, nationality) are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated. The paper says 77 students but does not explain how they were selected from the course enrollment of 100+."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes 'Monte Carlo random assignment (Metropolis and Ulam, 1949) stratified by gender' to assign 39 students to experimental and 38 to control groups."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Students likely knew whether they had chatbot access or not, and no mention is made of evaluator blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is reported. The course has 100+ students but only 77 participated; this gap is unexplained."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The system uses Together AI API for Llama 3.3 70B but no API costs, tokens consumed, or latency per query are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "An A100 80GB is mentioned as the development server but no total compute budget, GPU hours, or API spending is stated."
       }
     }
   },
   "claims": [
     {
       "claim": "CyberBOT achieves an average BERTScore of 0.933 and Context Recall of 0.994 on the CyberQ dataset.",
       "evidence": "Table 1 reports these metrics across ZS, FS, and OD categories.",
       "supported": "moderate"
     },
     {
       "claim": "The ontology-based validation reduces the risk of misleading or unsafe guidance.",
       "evidence": "No ablation comparing system with and without ontology validation. Only the architecture description in Section 2.3.",
       "supported": "unsupported"
     },
     {
       "claim": "The framework produces higher scores in the Few-shot category than Zero-shot and Ontology-Driven categories.",
       "evidence": "Table 1 shows consistently higher metrics for FS across most metrics (e.g., ROUGE-1: 0.788 FS vs 0.649 ZS).",
       "supported": "moderate"
     },
     {
       "claim": "CyberBOT has been deployed in a live classroom with more than 100 graduate students.",
       "evidence": "Section 4 describes deployment in CSE 546 at ASU for Spring 2025.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "CyberBOT is an ontology-grounded RAG system for cybersecurity education that combines an intent interpreter, retrieval-augmented generation, and ontology-based answer validation. Computational evaluation on the CyberQ dataset shows high BERTScore (0.933) and context recall (0.994), though no baselines are compared. A quasi-experimental study with 77 students is designed but results are not yet reported. The system is deployed in a live graduate course at ASU.",
   "red_flags": [
     {
       "flag": "No baselines",
       "detail": "The computational evaluation reports only CyberBOT's metrics without any baseline comparison (e.g., vanilla LLM, RAG without ontology). It is impossible to assess whether the ontology validation adds value."
     },
     {
       "flag": "No ablation study",
       "detail": "The system has three distinct components (intent interpreter, RAG, ontology verifier) but no ablation study measures their individual contributions."
     },
     {
       "flag": "Premature deployment claims",
       "detail": "The paper emphasizes real-world deployment and educational impact but the field study results are 'forthcoming.' The paper is essentially a system description with preliminary automated metrics."
     },
     {
       "flag": "Missing IRB approval",
       "detail": "A quasi-experimental study with 77 human participants collecting surveys and interview data is described without mentioning IRB or ethics board approval."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All metrics in Table 1 are point estimates without error bars, confidence intervals, or variance across runs."
     }
   ],
   "cited_papers": [
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper relevant to understanding retrieval-augmented generation approaches in AI systems."
     },
     {
       "title": "RAGAS: Automated evaluation of retrieval augmented generation",
       "authors": ["Shahul Es", "Jithin James", "Luis Espinosa Anke", "Steven Schockaert"],
       "year": 2024,
       "relevance": "RAG evaluation framework used in this paper, relevant to benchmarking and evaluation methodology for RAG systems."
     },
     {
       "title": "Seven failure points when engineering a retrieval augmented generation system",
       "authors": ["Scott Barnett", "Stefanus Kurniawan", "Srikanth Thudumu"],
       "year": 2024,
       "relevance": "Analysis of RAG system failure modes relevant to software engineering quality of AI systems."
     },
     {
       "title": "Mindful-RAG: A study of points of failure in retrieval augmented generation",
       "authors": ["Garima Agrawal", "Tharindu Kumarage", "Zeyad Alghamdi", "Huan Liu"],
       "year": 2024,
       "arxiv_id": "2407.12216",
       "relevance": "Study of RAG failure points relevant to reliability and safety of LLM-based systems."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Adversarial attacks on LLMs relevant to AI safety and security research."
     },
     {
       "title": "The wolf within: Covert injection of malice into MLLM societies via an MLLM operative",
       "authors": ["Zhen Tan", "Chengshuai Zhao", "Raha Moraffah"],
       "year": 2024,
       "arxiv_id": "2402.14859",
       "relevance": "Adversarial attacks on multi-modal LLM systems relevant to AI safety."
     },
     {
       "title": "\"Glue pizza and eat rocks\" - Exploiting vulnerabilities in retrieval-augmented generative models",
       "authors": ["Zhen Tan", "Chengshuai Zhao", "Raha Moraffah"],
       "year": 2024,
       "arxiv_id": "2406.19417",
       "relevance": "RAG vulnerability exploitation relevant to security of retrieval-augmented AI systems."
     },
     {
       "title": "Ontology-aware RAG for improved question-answering in cybersecurity education",
       "authors": ["Chengshuai Zhao", "Garima Agrawal", "Tharindu Kumarage"],
       "year": 2024,
       "arxiv_id": "2412.14191",
       "relevance": "Predecessor to CyberBOT, directly relevant to ontology-grounded RAG evaluation methodology."
     }
   ]
 }