scan.json (20083B)
1 { 2 "paper": { 3 "title": "Rethinking Knowledge Distillation in Collaborative Machine Learning: Memory, Knowledge, and Their Interactions", 4 "authors": ["Pengchao Han", "Xi Huang", "Yi Fang", "Guojun Han"], 5 "year": 2025, 6 "venue": "IEEE Transactions on Network Science and Engineering", 7 "arxiv_id": "2512.19972", 8 "doi": "10.1109/TNSE.2025.3572362" 9 }, 10 "scan_version": 2, 11 "active_modules": ["survey_methodology"], 12 "methodology_tags": ["meta-analysis"], 13 "key_findings": "This paper provides a comprehensive survey of knowledge distillation in collaborative learning, organized through the lens of memory and knowledge mechanisms. It categorizes memory (sensory, short-term, long-term) and knowledge (concrete vs. abstract, medium-separable vs. inseparable) in the KD process, then reviews KD across distributed, hierarchical, and decentralized collaborative patterns. The survey covers federated learning, domain adaptation, multi-modal learning, continual learning, multi-task learning, and knowledge graph embedding, with detailed tables summarizing the literature along dimensions of model/data/resource heterogeneity and privacy.", 14 "checklist": { 15 "artifacts": { 16 "code_released": { 17 "applies": true, 18 "answer": false, 19 "justification": "No code repository or analysis scripts are mentioned or linked in the paper." 20 }, 21 "data_released": { 22 "applies": true, 23 "answer": false, 24 "justification": "No dataset, corpus of reviewed papers, or extracted data tables are released. The survey could have released its literature corpus but did not." 25 }, 26 "environment_specified": { 27 "applies": false, 28 "answer": false, 29 "justification": "This is a survey paper with no computational experiments requiring an environment." 30 }, 31 "reproduction_instructions": { 32 "applies": true, 33 "answer": false, 34 "justification": "No instructions are provided for reproducing the literature search or the categorization presented in the paper." 35 } 36 }, 37 "statistical_methodology": { 38 "confidence_intervals_or_error_bars": { 39 "applies": false, 40 "answer": false, 41 "justification": "Survey paper with no experiments producing quantitative results." 42 }, 43 "significance_tests": { 44 "applies": false, 45 "answer": false, 46 "justification": "Survey paper with no statistical comparisons." 47 }, 48 "effect_sizes_reported": { 49 "applies": false, 50 "answer": false, 51 "justification": "Survey paper with no experiments." 52 }, 53 "sample_size_justified": { 54 "applies": false, 55 "answer": false, 56 "justification": "Survey paper with no experiments." 57 }, 58 "variance_reported": { 59 "applies": false, 60 "answer": false, 61 "justification": "Survey paper with no experiments." 62 } 63 }, 64 "evaluation_design": { 65 "baselines_included": { 66 "applies": true, 67 "answer": true, 68 "justification": "The paper explicitly compares itself against prior surveys (Section 1): Mora et al. [33] on KD in FL, Wu et al. [34] on KD for edge computing, Acharya et al. [35] on symbolic KD from LLMs, and Liu et al. [36] on crowd knowledge transfer. It distinguishes its contribution (memory/knowledge lens) from these prior works." 69 }, 70 "baselines_contemporary": { 71 "applies": true, 72 "answer": true, 73 "justification": "The compared prior surveys are from 2022-2024, which are recent and relevant." 74 }, 75 "ablation_study": { 76 "applies": false, 77 "answer": false, 78 "justification": "Survey paper with no system components to ablate." 79 }, 80 "multiple_metrics": { 81 "applies": false, 82 "answer": false, 83 "justification": "Survey paper with no quantitative evaluation." 84 }, 85 "human_evaluation": { 86 "applies": false, 87 "answer": false, 88 "justification": "Survey paper; human evaluation of system outputs is not relevant." 89 }, 90 "held_out_test_set": { 91 "applies": false, 92 "answer": false, 93 "justification": "Survey paper with no experiments." 94 }, 95 "per_category_breakdown": { 96 "applies": true, 97 "answer": true, 98 "justification": "The paper provides detailed breakdowns by collaborative pattern (distributed/hierarchical/decentralized), by task type (FL, MADA, FML, FCL, FMTL, FKGE), and by challenge dimension (model/data/resource heterogeneity, privacy). Tables 2-8 provide structured per-category comparisons." 99 }, 100 "failure_cases_discussed": { 101 "applies": true, 102 "answer": true, 103 "justification": "Section 9 discusses challenges and limitations of existing approaches, including tradeoffs between memory and knowledge (Sec 9.1), lack of proactive evaluation metrics (Sec 9.2), resource constraints (Sec 9.3), and limitations of decentralized KD (Sec 9.6)." 104 }, 105 "negative_results_reported": { 106 "applies": true, 107 "answer": true, 108 "justification": "The paper discusses known limitations: e.g., output sharing achieves lower accuracy than parameter sharing (Table 1), scale-down methods lead to performance decline (Section 2.1), and high KD temperature can dilute meaningful class distinctions (Section 2.2)." 109 } 110 }, 111 "claims_and_evidence": { 112 "abstract_claims_supported": { 113 "applies": true, 114 "answer": true, 115 "justification": "The abstract claims to provide a comprehensive review of KD in collaborative learning through memory/knowledge lenses, categorize memory and knowledge, examine collaborative patterns, and discuss challenges. All of these are substantively addressed in the paper body (Sections 2-9)." 116 }, 117 "causal_claims_justified": { 118 "applies": false, 119 "answer": false, 120 "justification": "The paper is a survey/review and does not make original causal claims. It summarizes findings from reviewed papers." 121 }, 122 "generalization_bounded": { 123 "applies": true, 124 "answer": false, 125 "justification": "The paper's title and abstract frame the work broadly as 'Collaborative Machine Learning' but the scope is heavily focused on federated learning and related variants. The paper does not explicitly bound its scope to these specific areas or acknowledge that other forms of collaborative ML (e.g., open-source collaboration, ensemble methods beyond KD) are excluded." 126 }, 127 "alternative_explanations_discussed": { 128 "applies": false, 129 "answer": false, 130 "justification": "Pure survey/taxonomy paper presenting no original empirical results that would require alternative explanations." 131 }, 132 "proxy_outcome_distinction": { 133 "applies": false, 134 "answer": false, 135 "justification": "Survey paper with no original measurements." 136 } 137 }, 138 "setup_transparency": { 139 "model_versions_specified": { 140 "applies": false, 141 "answer": false, 142 "justification": "Survey paper that does not use any models." 143 }, 144 "prompts_provided": { 145 "applies": false, 146 "answer": false, 147 "justification": "Survey paper that does not use prompting." 148 }, 149 "hyperparameters_reported": { 150 "applies": false, 151 "answer": false, 152 "justification": "Survey paper with no experiments." 153 }, 154 "scaffolding_described": { 155 "applies": false, 156 "answer": false, 157 "justification": "Survey paper with no agentic scaffolding." 158 }, 159 "data_preprocessing_documented": { 160 "applies": true, 161 "answer": false, 162 "justification": "The paper does not describe how papers were selected for inclusion. There is no description of search queries, databases searched, inclusion/exclusion criteria, or filtering pipeline for the literature review." 163 } 164 }, 165 "limitations_and_scope": { 166 "limitations_section_present": { 167 "applies": true, 168 "answer": true, 169 "justification": "Section 9 'Challenges & Future Directions' serves as a limitations discussion, covering gaps in evaluation metrics (9.2), resource constraints (9.3), memory management (9.4), incentive mechanisms (9.5), topology optimization (9.6), and theoretical analysis gaps (9.7)." 170 }, 171 "threats_to_validity_specific": { 172 "applies": true, 173 "answer": false, 174 "justification": "Section 9 discusses challenges of the field but not specific threats to the validity of this survey itself. There is no discussion of potential selection bias in the literature covered, completeness of the review, or potential mischaracterization of reviewed work." 175 }, 176 "scope_boundaries_stated": { 177 "applies": true, 178 "answer": false, 179 "justification": "The paper does not explicitly state what is excluded from its scope. It focuses on KD in collaborative learning but does not articulate boundaries — e.g., which types of collaborative learning or knowledge transfer are out of scope." 180 } 181 }, 182 "data_integrity": { 183 "raw_data_available": { 184 "applies": true, 185 "answer": false, 186 "justification": "No list of all reviewed papers, search results, or raw data underlying the categorization tables is available." 187 }, 188 "data_collection_described": { 189 "applies": true, 190 "answer": false, 191 "justification": "The paper does not describe how the surveyed literature was collected — no search queries, databases, date ranges, or systematic process is described." 192 }, 193 "recruitment_methods_described": { 194 "applies": false, 195 "answer": false, 196 "justification": "No human participants; data source is published literature (not a standard benchmark)." 197 }, 198 "data_pipeline_documented": { 199 "applies": true, 200 "answer": false, 201 "justification": "No documentation of how papers were identified, screened, and selected for inclusion in the review." 202 } 203 }, 204 "conflicts_of_interest": { 205 "funding_disclosed": { 206 "applies": true, 207 "answer": true, 208 "justification": "Funding is disclosed: National Natural Science Foundation of China (Grants 62401161, 62301336, 62322106, 62471151), Guangdong Basic and Applied Basic Research Foundation, Science and Technology Program of Guangzhou, and Longgang District project." 209 }, 210 "affiliations_disclosed": { 211 "applies": true, 212 "answer": true, 213 "justification": "Author affiliations are clearly disclosed: Guangdong University of Technology and Around Tech Company Ltd. / Shenzhen Institute of AI and Robotics for Society." 214 }, 215 "funder_independent_of_outcome": { 216 "applies": true, 217 "answer": true, 218 "justification": "Funding sources are government research grants (NSFC, Guangdong foundation, Guangzhou program) with no apparent stake in the survey's conclusions." 219 }, 220 "financial_interests_declared": { 221 "applies": true, 222 "answer": false, 223 "justification": "No competing interests or financial interests statement is included. One author is affiliated with Around Tech Company Ltd., but no disclosure of whether this creates conflicts." 224 } 225 }, 226 "contamination": { 227 "training_cutoff_stated": { 228 "applies": false, 229 "answer": false, 230 "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark." 231 }, 232 "train_test_overlap_discussed": { 233 "applies": false, 234 "answer": false, 235 "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark." 236 }, 237 "benchmark_contamination_addressed": { 238 "applies": false, 239 "answer": false, 240 "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark." 241 } 242 }, 243 "human_studies": { 244 "pre_registered": { 245 "applies": false, 246 "answer": false, 247 "justification": "No human participants." 248 }, 249 "irb_or_ethics_approval": { 250 "applies": false, 251 "answer": false, 252 "justification": "No human participants." 253 }, 254 "demographics_reported": { 255 "applies": false, 256 "answer": false, 257 "justification": "No human participants." 258 }, 259 "inclusion_exclusion_criteria": { 260 "applies": false, 261 "answer": false, 262 "justification": "No human participants." 263 }, 264 "randomization_described": { 265 "applies": false, 266 "answer": false, 267 "justification": "No human participants." 268 }, 269 "blinding_described": { 270 "applies": false, 271 "answer": false, 272 "justification": "No human participants." 273 }, 274 "attrition_reported": { 275 "applies": false, 276 "answer": false, 277 "justification": "No human participants." 278 } 279 }, 280 "cost_and_practicality": { 281 "inference_cost_reported": { 282 "applies": false, 283 "answer": false, 284 "justification": "Survey paper with no computational method to cost." 285 }, 286 "compute_budget_stated": { 287 "applies": false, 288 "answer": false, 289 "justification": "Survey paper with no computational experiments." 290 } 291 }, 292 "survey_methodology": { 293 "prisma_or_structured_protocol": { 294 "applies": true, 295 "answer": false, 296 "justification": "No PRISMA flow diagram, no registered protocol, no systematic search strategy with reproducible queries. The paper appears to collect literature ad hoc without a structured review protocol." 297 }, 298 "quality_assessment_of_sources": { 299 "applies": true, 300 "answer": false, 301 "justification": "The survey does not assess the methodological quality of the papers it reviews. All cited works are treated equally regardless of their experimental rigor, sample sizes, or reproducibility." 302 }, 303 "publication_bias_discussed": { 304 "applies": true, 305 "answer": false, 306 "justification": "No discussion of publication bias. The survey does not consider whether published KD results skew positive or whether negative results in collaborative KD are underrepresented." 307 } 308 } 309 }, 310 "claims": [ 311 { 312 "claim": "KD enables effective knowledge transfer among heterogeneous agents in collaborative learning by operating on standardized model outputs regardless of differing architectures.", 313 "evidence": "Section 6.1.1 reviews numerous methods (FedMD, DS-FL, FedDF, etc.) that handle model heterogeneity through output sharing. Table 1 compares output sharing vs parameter sharing approaches.", 314 "supported": "moderate" 315 }, 316 { 317 "claim": "Output sharing (black-box KD) offers advantages in privacy, communication efficiency, and heterogeneous model support, but typically achieves lower accuracy than parameter sharing.", 318 "evidence": "Table 1 provides a structured comparison showing output sharing has lower communication/computation cost and higher privacy but lower accuracy than parameter sharing.", 319 "supported": "moderate" 320 }, 321 { 322 "claim": "Memory and knowledge in KD can be systematically categorized using an Atkinson-Shiffrin-inspired framework (sensory, short-term, long-term memory) and knowledge dimensions (medium-separability, locality, semantic property).", 323 "evidence": "Sections 3.1-3.3 present the categorization framework with Figure 3 illustrating the memory hierarchy. These are definitional contributions rather than empirically validated claims.", 324 "supported": "weak" 325 }, 326 { 327 "claim": "Decentralized KD approaches remain limited, primarily focusing on fixed topology designs without adaptive topology optimization.", 328 "evidence": "Table 8 shows all reviewed decentralized KD works use fixed (non-changing) topologies. Section 9.6 identifies adaptive topology as an open challenge.", 329 "supported": "moderate" 330 } 331 ], 332 "red_flags": [ 333 { 334 "flag": "No systematic review protocol", 335 "detail": "For a comprehensive survey claiming to review KD in collaborative learning, there is no description of search methodology, databases queried, inclusion/exclusion criteria, or PRISMA-like protocol. The paper collection appears ad hoc, making it impossible to assess completeness or selection bias." 336 }, 337 { 338 "flag": "No quality assessment of reviewed papers", 339 "detail": "The survey treats all reviewed works equally without assessing their experimental rigor. Claims from papers with controlled experiments are weighted the same as claims from papers with limited evaluation. This risks laundering weak results into the survey's conclusions." 340 }, 341 { 342 "flag": "Conceptual framework not empirically validated", 343 "detail": "The memory/knowledge categorization framework (Sections 3-4) is presented as a novel contribution but is based on analogy to cognitive science models (Atkinson-Shiffrin) without empirical validation that these categories are useful, complete, or predictive in the ML context." 344 }, 345 { 346 "flag": "Suspicious manuscript date", 347 "detail": "The manuscript metadata states 'Manuscript received April 19, 2005; revised August 26, 2015' which appears to be a LaTeX template artifact, not the actual submission dates." 348 } 349 ], 350 "cited_papers": [ 351 { 352 "title": "Distilling the knowledge in a neural network", 353 "authors": ["G. Hinton", "O. Vinyals", "J. Dean"], 354 "year": 2015, 355 "arxiv_id": "1503.02531", 356 "relevance": "Foundational KD paper; relevant to understanding knowledge transfer mechanisms in AI systems." 357 }, 358 { 359 "title": "Communication-efficient learning of deep networks from decentralized data", 360 "authors": ["B. McMahan", "E. Moore", "D. Ramage", "S. Hampson", "B. A. y Arcas"], 361 "year": 2017, 362 "relevance": "Introduces FedAvg for federated learning, the core distributed learning paradigm reviewed in this survey." 363 }, 364 { 365 "title": "Ensemble distillation for robust model fusion in federated learning", 366 "authors": ["T. Lin", "L. Kong", "S. U. Stich", "M. Jaggi"], 367 "year": 2021, 368 "relevance": "FedDF combines parameter averaging with KD for federated learning, a key method for model heterogeneity." 369 }, 370 { 371 "title": "Knowledge distillation for federated learning: a practical guide", 372 "authors": ["A. Mora", "I. Tenison", "P. Bellavista", "I. Rish"], 373 "year": 2022, 374 "arxiv_id": "2211.04742", 375 "relevance": "Prior survey on KD in federated learning; useful for comparison with survey methodology approaches." 376 }, 377 { 378 "title": "A survey on symbolic knowledge distillation of large language models", 379 "authors": ["K. Acharya", "A. Velasquez", "H. H. Song"], 380 "year": 2024, 381 "relevance": "Reviews symbolic KD from LLMs, relevant to understanding how LLM knowledge can be distilled into interpretable forms." 382 }, 383 { 384 "title": "Data-free knowledge distillation for heterogeneous federated learning", 385 "authors": ["Z. Zhu", "J. Hong", "J. Zhou"], 386 "year": 2021, 387 "relevance": "FedGen addresses data heterogeneity in FL through generated data for KD, relevant to understanding data-free knowledge transfer." 388 }, 389 { 390 "title": "Federated adversarial domain adaptation", 391 "authors": ["X. Peng", "Z. Huang", "Y. Zhu", "K. Saenko"], 392 "year": 2019, 393 "relevance": "FADA combines federated learning with domain adaptation using adversarial learning, relevant to multi-agent knowledge transfer." 394 }, 395 { 396 "title": "GPT-4 technical report", 397 "authors": ["J. Achiam", "S. Adler", "S. Agarwal"], 398 "year": 2023, 399 "arxiv_id": "2303.08774", 400 "relevance": "Foundation model whose deployment motivates scale-down KD methods reviewed in this survey." 401 } 402 ] 403 }