ai-research-survey

scan.json (23227B)

---

 {
   "paper": {
     "title": "Beyond Textual Context: Structural Graph Encoding with Adaptive Space Alignment to alleviate the hallucination of LLMs",
     "authors": ["Yifang Zhang", "Pengfei Duan", "Yiwen Yang", "Shengwu Xiong"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.22251"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code are available at https://github.com/yfangZhang/SSKG-LLM', providing a GitHub repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmark datasets: CommonsenseQA, SIQA, and TruthfulQA. These are standard public benchmarks the authors did not modify."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'a NVIDIA GeForce RTX 5880 GPU (with 48GB of VRAM)' and an AdamW optimizer with learning rate 1e-4, but provides no requirements.txt, Dockerfile, conda environment, or detailed library version list sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A GitHub link is given but the paper itself does not include a 'Reproducing Results' section or commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1 and 2 and Figures 3-5 are reported as point estimates without confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SSKG-LLM 'significantly outperforms' baselines (Section 4.3) but provides no statistical significance tests (no p-values, t-tests, or any other test). Differences are assessed purely by comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'improvements across different datasets are approximately 8%' (Section 4.3) but does not provide systematic effect sizes with baseline context for each comparison. No Cohen's d, odds ratios, or other formal effect size measures are reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of datasets or their sizes. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. All results appear to be single-run numbers. The robustness analysis in the appendix varies LoRA parameters but does not report variance across repeated runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes extensive baselines across four categories: base LLMs (no retrieval), RAG methods, LLMs-based KBQA (GPT-3.5+KSL, ChatGPT+ToG-R, GPT-4+ToG-R, KG-CoT), and hybrid encoding methods (KnowLA, KAPING, KG-Adapter)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods from 2023-2024: KG-CoT (2024), KnowLA (2024), KG-Adapter (2024), ToG-R (2024), and others. These represent the state of the art in KG-LLM integration."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents an ablation study systematically removing each module (KGR, KGE, KGA) to assess individual contributions. Additional ablations are provided for graph traversal strategies (Section 4.4.3) and different KG encoders (Section 4.4.3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses accuracy (ACC) for CommonsenseQA and SIQA, and ROUGE-1, ROUGE-2, and BLEU-score for TruthfulQA, providing multiple metrics across tasks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluations are automated (accuracy matching and similarity metrics). For a paper claiming to reduce hallucinations and improve truthfulness, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly describe the separation of dev and test splits. It mentions using standard datasets but does not clarify whether results are on held-out test sets or validation sets used for any tuning decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (CommonsenseQA, SIQA, TruthfulQA) in Table 1 and per module configuration in Table 2. Additional per-encoder and per-traversal breakdowns are in Figures 3-4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or qualitative error analysis is provided. The paper does not show examples where SSKG-LLM produces incorrect answers or discuss specific failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study reveals a negative finding: removing the KGA module while keeping KGE actually hurts performance compared to removing both (Section 4.4.2), showing that the KG encoder without adaptation exacerbates the embedding gap. This is an informative negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims SSKG-LLM integrates structural and semantic information of KGs and enhances factual reasoning. Table 1 shows improvements over baselines on all three datasets, and the ablation study supports the contribution of each module. The claims are supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies ('removing component X reduces performance by Y%' in Table 2 and Sections 4.4.1-4.4.3). The ablation design uses controlled single-variable manipulation, which is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract make broad claims about alleviating 'the hallucination of LLMs' but testing is limited to three English QA datasets with two specific LLMs (Qwen1.5-14B and Llama2-13B). The Limitation section acknowledges the English-only constraint but the title/abstract do not bound generalization to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For instance, improvements could be partially due to additional parameters from the KGA module or the fine-tuning process rather than structural KG information specifically. No threats-to-validity or confound discussion is included."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Qwen1.5-14B-Chat' and 'Llama2-13B-Chat' as the foundation models (Section 4.2), along with the GraphLM encoder. These are specific enough to identify the exact model variants."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in the traditional sense. The input to the LLM is constructed via an embedding pipeline (KGR -> KGE -> KGA -> concatenation with query embedding), not through text prompts. The system is fine-tuned end-to-end with LoRA."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports: learning rate 1e-4, linear warm-up for 100 steps, batch size 4, AdamW optimizer. The appendix explores LoRA rank values R=8, R=16, R=32. GPU type (RTX 5880, 48GB VRAM) is also stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. SSKG-LLM is a model architecture with modular components (KGR, KGE, KGA) that processes input in a single forward pass, not an agent with tools, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the KGR preprocessing pipeline: tokenization via SpaCy, stopword filtering via NLTK, keyword mapping to ConceptNet via exact string matching, TF-IDF scoring for topic identification, subgraph retrieval via Wikidata/ConceptNet APIs, and DFS serialization."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitation' section is present after the Conclusion (Section 5), discussing computational resource demands and the English-only constraint."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions two generic points: 'more computational resources' and 'limited to English datasets.' These are surface-level observations without specific quantification or analysis of how they affect the reported results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The Limitation section mentions English-only testing and computational cost but does not bound the claims to specific KG types, domains, question formats, or model scales."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmark datasets are publicly available, the paper does not release the actual experimental outputs, model predictions, or training data constructed for fine-tuning."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the datasets used (CommonsenseQA, SIQA, TruthfulQA) with citations. Section 3.1 describes how knowledge graphs are retrieved from ConceptNet and Wikidata. The data collection procedure for the benchmark evaluation is adequately documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper uses standard benchmarks (CommonsenseQA, SIQA, TruthfulQA) which are publicly available."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from query to answer is documented in Section 3: tokenization -> topic extraction -> subgraph retrieval -> DFS serialization -> Levi graph conversion -> GraphLM encoding -> KGA adaptation -> concatenation -> LoRA fine-tuned LLM inference."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: all four authors are from Wuhan University of Technology. Correspondence emails are provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding statement does not establish that the work is unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper fine-tunes Qwen1.5-14B-Chat and Llama2-13B-Chat on QA benchmarks but does not state the training data cutoff dates of these base models. CommonsenseQA (2019) and SIQA (2019) could be in the pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap between the pre-training data of Qwen1.5/Llama2 and the benchmark test sets. CommonsenseQA and SIQA are well-known benchmarks that may appear in pre-training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CommonsenseQA (2019), SIQA (2019), and TruthfulQA (2022) were all published before the training cutoffs of Qwen1.5 and Llama2, creating contamination risk. This is not addressed in the paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens-per-example is reported. The Limitation section acknowledges the method 'demands more computational resources' but provides no quantification."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the GPU type (RTX 5880, 48GB VRAM) is mentioned, the total training time, GPU hours, or total computational budget is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "SSKG-LLM consistently outperforms all baseline methods across both multiple-choice QA datasets and the TruthfulQA dataset, with improvements of approximately 8%.",
       "evidence": "Table 1 shows SSKG-LLM (Llama2-13B) achieves 86.54% on CommonsenseQA vs 85.65% best baseline (SFT no-retrieval), 77.92% on SIQA vs 77.8% (ChatGPT+ToG-R), and 0.3462 BLEU on TruthfulQA vs 0.3391 best baseline.",
       "supported": "weak"
     },
     {
       "claim": "The KGA module bridges the gap between KG encoder embeddings and LLM embedding spaces, and removing it leads to performance decline.",
       "evidence": "Table 2 ablation shows removing KGA (row 1: KGR+KGE without KGA) reduces SIQA accuracy from 77.95% to 70.77%, a 7.18% drop. Section 4.4.2 discusses the embedding gap phenomenon.",
       "supported": "moderate"
     },
     {
       "claim": "DFS traversal strategy is superior to BFS and Random Walk for KG serialization.",
       "evidence": "Figure 3 shows DFS outperforms BFS and Random Walk across datasets, attributed to generating longer chains similar to Chain-of-Thought reasoning (Section 4.4.3).",
       "supported": "moderate"
     },
     {
       "claim": "GraphLM consistently delivers superior performance compared to GNN, GraphTransformer, and no-encoder baselines.",
       "evidence": "Figure 4 shows GraphLM outperforms alternatives across datasets (Section 4.4.3).",
       "supported": "moderate"
     },
     {
       "claim": "SSKG-LLM is the first approach to simultaneously integrate both semantic and structural information of KGs during joint reasoning with LLMs.",
       "evidence": "Stated in contributions (Section 1). The related work (Section 2) categorizes prior methods into Hybrid Encoding, GraphRAG, and LLMs-based KBQA, arguing each has limitations in this regard.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SSKG-LLM integrates structural and semantic information from knowledge graphs into LLM reasoning through three modules: KGR (retrieval), KGE (encoding with GraphLM), and KGA (cross-attention adaptation). Evaluated on CommonsenseQA, SIQA, and TruthfulQA, SSKG-LLM shows modest improvements over baselines (approximately 0.7-1% over the best SFT baselines on multiple-choice tasks). Ablation studies reveal that removing the KGA module causes significant performance drops, particularly on SIQA (7.18%), and that directly injecting KG embeddings without adaptation can hurt performance compared to plain-text input. The DFS traversal strategy and GraphLM encoder are shown to be optimal choices within the framework.",
   "red_flags": [
     {
       "flag": "Overstated improvement claims",
       "detail": "The paper claims 'approximately 8%' improvement but actual margins over the best SFT baselines are much smaller: ~0.9% on CommonsenseQA (86.54 vs 85.65), ~0.1% on SIQA (77.92 vs 77.8), and ~0.07 BLEU on TruthfulQA (0.3462 vs 0.3391). The 8% figure seems to compare against non-SFT or weaker baselines selectively."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Given the small margins over the best baselines (often <1%), the absence of significance tests, error bars, or multi-run variance reporting makes it impossible to determine whether the claimed improvements are meaningful or within noise."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "CommonsenseQA (2019) and SIQA (2019) are old enough to be in the pre-training data of Qwen1.5 and Llama2. The fine-tuning process could be overfitting to memorized patterns rather than demonstrating genuine structural reasoning improvement."
     },
     {
       "flag": "Incomplete baseline comparisons",
       "detail": "Many baseline entries in Table 1 have missing values (dashes) across datasets. GPT-3.5+KSL has results only for CommonsenseQA, KG-CoT only for SIQA, KG-Adapter only for CommonsenseQA. This makes cross-method comparison difficult and potentially misleading."
     },
     {
       "flag": "No error bars on single-run results",
       "detail": "Results appear to be from single runs with no repeated experiments. For the small margins reported, variance across random seeds or data ordering could easily explain the differences."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 Technical Report",
       "authors": ["Josh Achiam"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM evaluated in the baseline comparisons, central to the LLM capability assessment literature."
     },
     {
       "title": "From Local to Global: A Graph RAG Approach to Query-Focused Summarization",
       "authors": ["Darren Edge", "Ha Trinh", "Newman Cheng"],
       "year": 2024,
       "arxiv_id": "2404.16130",
       "relevance": "Key GraphRAG method for integrating knowledge graphs with LLMs, directly relevant to retrieval-augmented generation approaches."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["Edward J Hu"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning method used as the core training approach in SSKG-LLM and widely used across LLM research."
     },
     {
       "title": "Think-on-Graph: Deep and Responsible Reasoning of Large Language Model on Knowledge Graph",
       "authors": ["Jiashuo Sun", "Chengjin Xu"],
       "year": 2024,
       "relevance": "LLM-based KBQA method used as a baseline, demonstrates agentic LLM reasoning over knowledge graphs."
     },
     {
       "title": "KG-CoT: Chain-of-Thought Prompting of Large Language Models over Knowledge Graphs for Knowledge-Aware Question Answering",
       "authors": ["Ruilin Zhao", "Feng Zhao"],
       "year": 2024,
       "relevance": "Baseline method combining chain-of-thought reasoning with knowledge graphs, relevant to LLM reasoning enhancement."
     },
     {
       "title": "KnowLA: Enhancing Parameter-Efficient Finetuning with Knowledgeable Adaptation",
       "authors": ["Xindi Luo", "Zequn Sun"],
       "year": 2024,
       "relevance": "Hybrid encoding baseline that integrates KG entity embeddings into LLMs via adaptation layers."
     },
     {
       "title": "KG-Adapter: Enabling Knowledge Graph Integration in Large Language Models through Parameter-Efficient Fine-Tuning",
       "authors": ["Shiyu Tian", "Yangyang Luo"],
       "year": 2024,
       "relevance": "Parameter-level KG integration method used as a baseline, directly comparable approach to SSKG-LLM."
     },
     {
       "title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2022,
       "relevance": "Key benchmark for evaluating LLM truthfulness and hallucination, used as evaluation dataset in this paper."
     },
     {
       "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models",
       "authors": ["Hugo Touvron", "Louis Martin"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "One of two foundation LLMs used in experiments, central to open-source LLM evaluation."
     },
     {
       "title": "Unifying Large Language Models and Knowledge Graphs: A Roadmap",
       "authors": ["Shirui Pan", "Linhao Luo"],
       "year": 2024,
       "relevance": "Survey paper on LLM-KG integration approaches, provides context for the research area."
     }
   ]
 }