scan.json (15521B)
1 { 2 "paper": { 3 "title": "Cyber-Physical System Defense Against Structured False Data Injection Attacks Using an Adaptive Security Framework with Passivity Enhancement", 4 "authors": ["Gopi R", "Venkatesh S", "Francis Shamili S", "Parthiban K", "Jagadeeswari S", "Arulmozhi P", "Suganya B"], 5 "year": 2025, 6 "venue": "Journal of Information Systems Engineering and Management", 7 "doi": null 8 }, 9 "checklist": { 10 "artifacts": { 11 "code_released": { 12 "applies": true, 13 "answer": false, 14 "justification": "No source code repository or archive is provided. Algorithm 1 is pseudocode only." 15 }, 16 "data_released": { 17 "applies": true, 18 "answer": true, 19 "justification": "A Kaggle dataset link is provided: https://www.kaggle.com/datasets/teamincribo/cyber-security-attacks" 20 }, 21 "environment_specified": { 22 "applies": true, 23 "answer": false, 24 "justification": "No environment specifications, library versions, or dependency information provided." 25 }, 26 "reproduction_instructions": { 27 "applies": true, 28 "answer": false, 29 "justification": "No reproduction instructions or steps to replicate the experiments are provided." 30 } 31 }, 32 "statistical_methodology": { 33 "confidence_intervals_or_error_bars": { 34 "applies": true, 35 "answer": false, 36 "justification": "Table 2 reports point estimates only (e.g., 97.5% accuracy) with no confidence intervals or error bars." 37 }, 38 "significance_tests": { 39 "applies": true, 40 "answer": false, 41 "justification": "Claims of improvement over traditional methods are made without any statistical significance tests." 42 }, 43 "effect_sizes_reported": { 44 "applies": true, 45 "answer": true, 46 "justification": "Table 2 reports percentage improvements with baseline context (e.g., 97.5% vs 85.2%, +14.4% improvement)." 47 }, 48 "sample_size_justified": { 49 "applies": true, 50 "answer": false, 51 "justification": "No discussion of sample size, dataset size used for evaluation, or justification for experimental scale." 52 }, 53 "variance_reported": { 54 "applies": true, 55 "answer": false, 56 "justification": "No variance, standard deviation, or spread measures reported across experimental runs." 57 } 58 }, 59 "evaluation_design": { 60 "baselines_included": { 61 "applies": true, 62 "answer": true, 63 "justification": "Table 2 compares PEASF against 'Traditional Methods' on multiple metrics." 64 }, 65 "baselines_contemporary": { 66 "applies": true, 67 "answer": false, 68 "justification": "The baseline is labeled generically as 'Traditional Methods' without specifying which methods or their recency. The literature survey mentions SADS, CCSC, IISSF, MCAMD but these are not used as baselines in experiments." 69 }, 70 "ablation_study": { 71 "applies": true, 72 "answer": false, 73 "justification": "PEASF has multiple components (Kalman filtering, ML classifiers, graph models, passivity-based control) but no ablation study examines their individual contributions." 74 }, 75 "multiple_metrics": { 76 "applies": true, 77 "answer": true, 78 "justification": "Table 2 reports detection accuracy, false positive rate, system stability (recovery time), computational efficiency (latency), and CPU usage." 79 }, 80 "human_evaluation": { 81 "applies": false, 82 "answer": false, 83 "justification": "Human evaluation is irrelevant to CPS attack detection framework evaluation." 84 }, 85 "held_out_test_set": { 86 "applies": true, 87 "answer": false, 88 "justification": "No mention of train/test splits or held-out evaluation data." 89 }, 90 "per_category_breakdown": { 91 "applies": true, 92 "answer": false, 93 "justification": "No breakdown by attack type, scenario, or category — only aggregate metrics in Table 2." 94 }, 95 "failure_cases_discussed": { 96 "applies": true, 97 "answer": false, 98 "justification": "No failure cases or error analysis discussed. Only positive results presented." 99 }, 100 "negative_results_reported": { 101 "applies": true, 102 "answer": false, 103 "justification": "Every reported metric shows improvement. No negative results or failed configurations reported." 104 } 105 }, 106 "claims_and_evidence": { 107 "abstract_claims_supported": { 108 "applies": true, 109 "answer": false, 110 "justification": "The abstract claims PEASF 'significantly enhances system stability, reduces detection errors, and enhances security resilience' but the evidence is a single table of round numbers with no statistical backing or experimental detail." 111 }, 112 "causal_claims_justified": { 113 "applies": true, 114 "answer": false, 115 "justification": "The paper claims PEASF causes improvements ('enhances', 'reduces') but there is no controlled experimental design or adequate ablation to justify causal claims." 116 }, 117 "generalization_bounded": { 118 "applies": true, 119 "answer": false, 120 "justification": "The paper claims applicability to 'smart grids, intelligent transportation systems, and industrial automation' but tests only on a generic Kaggle cybersecurity dataset with no domain-specific validation." 121 }, 122 "alternative_explanations_discussed": { 123 "applies": true, 124 "answer": false, 125 "justification": "No alternative explanations for the results are discussed." 126 } 127 }, 128 "setup_transparency": { 129 "model_versions_specified": { 130 "applies": true, 131 "answer": false, 132 "justification": "The paper mentions 'machine learning classifiers' without specifying which classifiers, versions, or configurations." 133 }, 134 "prompts_provided": { 135 "applies": false, 136 "answer": false, 137 "justification": "This paper does not use LLM prompting." 138 }, 139 "hyperparameters_reported": { 140 "applies": true, 141 "answer": false, 142 "justification": "No hyperparameters reported for the ML classifiers, Kalman filter parameters, or detection thresholds used in experiments." 143 }, 144 "scaffolding_described": { 145 "applies": false, 146 "answer": false, 147 "justification": "No agentic scaffolding is used in this paper." 148 }, 149 "data_preprocessing_documented": { 150 "applies": true, 151 "answer": false, 152 "justification": "No description of how the Kaggle dataset was preprocessed, filtered, or adapted for the CPS simulation testbed." 153 } 154 }, 155 "limitations_and_scope": { 156 "limitations_section_present": { 157 "applies": true, 158 "answer": true, 159 "justification": "The conclusion mentions: 'A limitation of this work is that it trains on pre-existing attack models, which may limit its ability to be applied to threats that are continuously evolving and highly novel.'" 160 }, 161 "threats_to_validity_specific": { 162 "applies": true, 163 "answer": false, 164 "justification": "The single limitation sentence is generic. No specific threats to validity are discussed (e.g., dataset representativeness, simulation fidelity)." 165 }, 166 "scope_boundaries_stated": { 167 "applies": true, 168 "answer": false, 169 "justification": "No explicit boundaries on what the results do NOT show. The paper broadly claims applicability across multiple CPS domains." 170 } 171 }, 172 "data_integrity": { 173 "raw_data_available": { 174 "applies": true, 175 "answer": true, 176 "justification": "The Kaggle dataset link is provided for the source data." 177 }, 178 "data_collection_described": { 179 "applies": true, 180 "answer": false, 181 "justification": "The paper states 'In the dataset link [12], the values are chosen to describe the strength of the proposed method' but does not describe how the dataset was used or what data collection procedure was followed." 182 }, 183 "recruitment_methods_described": { 184 "applies": false, 185 "answer": false, 186 "justification": "No human participants; uses a public dataset." 187 }, 188 "data_pipeline_documented": { 189 "applies": true, 190 "answer": false, 191 "justification": "No documentation of how data flows from the Kaggle dataset to the simulation results in Table 2." 192 } 193 }, 194 "conflicts_of_interest": { 195 "funding_disclosed": { 196 "applies": true, 197 "answer": true, 198 "justification": "The paper states: 'The authors received no specific funding for this study.'" 199 }, 200 "affiliations_disclosed": { 201 "applies": true, 202 "answer": true, 203 "justification": "All seven authors list their academic affiliations and departments." 204 }, 205 "funder_independent_of_outcome": { 206 "applies": false, 207 "answer": false, 208 "justification": "The study is unfunded." 209 }, 210 "financial_interests_declared": { 211 "applies": true, 212 "answer": true, 213 "justification": "The paper includes a conflicts of interest statement declaring no financial or personal relationships that could influence the research." 214 } 215 }, 216 "contamination": { 217 "training_cutoff_stated": { 218 "applies": false, 219 "answer": false, 220 "justification": "This paper does not evaluate a pre-trained LLM on a benchmark." 221 }, 222 "train_test_overlap_discussed": { 223 "applies": false, 224 "answer": false, 225 "justification": "This paper does not evaluate a pre-trained LLM on a benchmark." 226 }, 227 "benchmark_contamination_addressed": { 228 "applies": false, 229 "answer": false, 230 "justification": "This paper does not evaluate a pre-trained LLM on a benchmark." 231 } 232 }, 233 "human_studies": { 234 "pre_registered": { 235 "applies": false, 236 "answer": false, 237 "justification": "No human participants in this study." 238 }, 239 "irb_or_ethics_approval": { 240 "applies": false, 241 "answer": false, 242 "justification": "No human participants in this study." 243 }, 244 "demographics_reported": { 245 "applies": false, 246 "answer": false, 247 "justification": "No human participants in this study." 248 }, 249 "inclusion_exclusion_criteria": { 250 "applies": false, 251 "answer": false, 252 "justification": "No human participants in this study." 253 }, 254 "randomization_described": { 255 "applies": false, 256 "answer": false, 257 "justification": "No human participants in this study." 258 }, 259 "blinding_described": { 260 "applies": false, 261 "answer": false, 262 "justification": "No human participants in this study." 263 }, 264 "attrition_reported": { 265 "applies": false, 266 "answer": false, 267 "justification": "No human participants in this study." 268 } 269 }, 270 "cost_and_practicality": { 271 "inference_cost_reported": { 272 "applies": true, 273 "answer": true, 274 "justification": "Table 2 reports detection latency (50ms vs 180ms) and CPU usage (35% vs 70%)." 275 }, 276 "compute_budget_stated": { 277 "applies": true, 278 "answer": false, 279 "justification": "No total computational budget, hardware specifications, or simulation runtime reported." 280 } 281 } 282 }, 283 "claims": [ 284 { 285 "claim": "PEASF achieves 97.5% attack detection accuracy compared to 85.2% for traditional methods (+14.4% improvement)", 286 "evidence": "Table 2 in Results and Discussion section", 287 "supported": "weak" 288 }, 289 { 290 "claim": "PEASF reduces false positive rate to 2.3% from 8.6% (-73.3%)", 291 "evidence": "Table 2 in Results and Discussion section", 292 "supported": "weak" 293 }, 294 { 295 "claim": "PEASF achieves 150ms recovery time vs 400ms for traditional methods (+62.5% faster)", 296 "evidence": "Table 2 in Results and Discussion section", 297 "supported": "weak" 298 }, 299 { 300 "claim": "PEASF reduces CPU usage to 35% from 70% (-50%)", 301 "evidence": "Table 2 in Results and Discussion section", 302 "supported": "weak" 303 } 304 ], 305 "methodology_tags": ["benchmark-eval"], 306 "key_findings": "The paper proposes PEASF, a framework combining passivity-based control, Kalman filtering, ML classifiers, and graph models to defend CPS against False Data Injection Attacks. The authors report 97.5% detection accuracy, 2.3% false positive rate, and significant computational efficiency gains over unnamed 'traditional methods.' However, the experimental methodology is severely underdocumented — no details on simulation setup, ML models used, hyperparameters, or statistical validation are provided.", 307 "red_flags": [ 308 { 309 "flag": "Suspiciously round and clean results", 310 "detail": "Table 2 values (97.5%, 2.3%, 150ms, 50ms, 35%) appear fabricated or hand-picked. The paper states 'the values are chosen to describe the strength of the proposed method,' which suggests the numbers may not come from actual experiments." 311 }, 312 { 313 "flag": "Meaningless equations", 314 "detail": "Equations 1-10 use arbitrary mathematical notation with undefined variables and no derivations. They appear decorative rather than functional — e.g., Eq. 1: '∂s′ = zvt + R[a + nr″] * V[σμ + πτϑ″] − δ[a − b′]' with no rigorous definition of terms." 315 }, 316 { 317 "flag": "Unnamed baselines", 318 "detail": "Comparisons are made against generic 'Traditional Methods' without specifying what they are. The literature review discusses SADS, CCSC, IISSF, MCAMD but none appear as experimental baselines." 319 }, 320 { 321 "flag": "Dataset mismatch", 322 "detail": "The linked Kaggle dataset (teamincribo/cyber-security-attacks) is a generic cybersecurity dataset, not a CPS/SCADA/smart grid dataset. It is unclear how this was adapted for the claimed CPS simulation testbed." 323 }, 324 { 325 "flag": "No experimental methodology", 326 "detail": "The paper provides no simulation details, no ML model specifications, no training procedure, and no experimental protocol. It is impossible to determine how the results in Table 2 were obtained." 327 } 328 ], 329 "cited_papers": [ 330 { 331 "title": "A novel passive-active detection system for false data injection attacks in industrial control systems", 332 "authors": ["Ma, Y. W.", "Tsou, C. W."], 333 "year": 2024, 334 "doi": "10.1016/j.cose.2024.103996", 335 "relevance": "Addresses FDIA detection in industrial control systems, relevant to CPS security methodology." 336 }, 337 { 338 "title": "Cyber-Physical Security Vulnerabilities Identification and Classification in Smart Manufacturing--A Defense-in-Depth Driven Framework and Taxonomy", 339 "authors": ["Rahman, M. H.", "Shafae, M."], 340 "year": 2024, 341 "arxiv_id": "2501.09023", 342 "relevance": "Taxonomy of CPS security vulnerabilities relevant to understanding defense frameworks." 343 }, 344 { 345 "title": "Security Control of Cyber-Physical Systems under Cyber Attacks: A Survey", 346 "authors": ["Xing, W.", "Shen, J."], 347 "year": 2024, 348 "doi": "10.3390/s24123815", 349 "relevance": "Survey of CPS security control methods, relevant to understanding the landscape of defense approaches." 350 } 351 ] 352 }