ai-research-survey

scan-v5.json (27156B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "GLM-Dialog: Noise-tolerant Pre-training for Knowledge-grounded Dialogue Generation",
     "authors": [
       "Jing Zhang",
       "Xiaokang Zhang",
       "Daniel Zhang-Li",
       "Jifan Yu",
       "Zijun Yao",
       "Zeyao Ma",
       "Yiqi Xu",
       "Haohua Wang",
       "Xiaohan Zhang",
       "Nianyi Lin",
       "Sunrui Lu",
       "Juanzi Li",
       "Jie Tang"
     ],
     "year": 2023,
     "venue": "Knowledge Discovery and Data Mining",
     "arxiv_id": "2302.14401",
     "doi": "10.1145/3580305.3599832"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GLM-Dialog outperforms open-source Chinese dialogue models, supported by Tables 1-3 showing consistent improvements on automatic metrics and human evaluation. The evaluation platform claim is supported by the online deployment described in Section 4.3.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies in Section 5.3 systematically remove components (stage-2 training, knowledge injection, knowledge classification, iterative injection) and measure resulting performance drops, providing adequate support for causal claims about each component's contribution.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are appropriately bounded to Chinese-language open-domain dialogue; the paper explicitly positions contributions relative to Chinese language community challenges and only evaluates on Chinese benchmarks.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether performance gains could be attributed to the GLM10B backbone quality, the larger total training data volume, or the search engine itself independent of the noise-tolerant training strategy.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges automatic metrics cannot faithfully reflect dialogue quality (Section 4.3) and supplements with human evaluation across multiple dimensions including coherence, informativeness, hallucination, and engagingness.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section; the only limitation acknowledged is hallucination mentioned briefly in Section 5.2 as speculation, and the conclusion lacks any formal limitations discussion.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed; the potential confound of DuSinc data appearing in both training and the DuSincR test set is never addressed, nor are annotator qualification or preference biases beyond the implicit evaluation design.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated regarding what the results do not show; the paper does not clarify limitations to Chinese, to specific dialogue types, or to the tested model scale.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper; there is no acknowledgments section mentioning grants or institutional support.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed: Renmin University of China, Tsinghua University, and Zhipu.AI, the company that commercializes GLM.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "One author (Xiaohan Zhang) is affiliated with Zhipu.AI, which commercializes GLM—the backbone evaluated favorably in this paper—creating a non-independent relationship between institutional affiliation and outcome.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present despite an author's affiliation with Zhipu.AI, a commercial entity with direct interest in GLM's perceived quality.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "The paper formally defines Dialogue History, External Knowledge Pool, and the knowledge-grounded dialogue generation task with mathematical notation in Section 2.2, and distinguishes helpful vs. noisy knowledge with concrete examples in Figure 1.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 clearly enumerates four contributions: noise-tolerant training techniques, an evaluation platform, an open-source dialogue model, and an easy-to-use toolkit for Chinese knowledge-grounded dialogue.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 contextualizes GLM-Dialog relative to LaMDA, GODEL, BlenderBot3, CDial-GPT, EVA2.0, and PLATO-XL, explicitly identifying the gap (limited high-quality non-English datasets) that this work addresses.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Source code is released on GitHub (https://github.com/RUCKBReasoning/GLM-Dialog) and model checkpoints are explicitly stated as released.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "Key training data (800K online service dialogues from XDAI) and the DuSincR evaluation benchmark are not clearly released for download; while public benchmarks are used for training, the paper's novel data contributions are not confirmed publicly available.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Only hardware (8×80G A100) and basic training parameters are specified; no requirements.txt, Dockerfile, or software environment specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided; the paper describes the methodology but a reader would need to infer the full pipeline from scattered descriptions across Sections 3.1-3.3.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are reported in any of the tables (Tables 1-5); only point estimates are given for all metrics.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are performed for any comparative claims despite multiple pairwise model comparisons across many metrics.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute metric values with baselines are reported in all tables, allowing readers to directly assess the magnitude of improvements (e.g., F1 22.010 vs. 13.548 for CDial-GPT).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The use of 100 conversations for ablation studies, 50+100 opening utterances for human evaluation, and 20 annotators for implicit evaluation are not justified with power analysis or sample size rationale.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or inter-annotator agreement statistics are reported for any human or automatic evaluation results.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Five baselines are included: CDial-GPT, EVA2.0, PLATO-2, GLM10B, and GLM130B, covering small, medium, and large models.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "The paper notes EVA and PLATO have newer versions but those lack released code; the selected baselines are the best available with accessible model weights as of early 2023.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Sections 5.3-5.4 present systematic ablation removing stage-2 training, knowledge injection, knowledge classification, iterative injection, and query generation with human evaluation results.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation uses six automatic metrics (BLEU-4, F1, Rouge-L, Rouge-1, Rouge-2, BertScore) and seven human metrics (coherence, informativeness, safety, inspiration, hallucination, engagingness, faithfulness).",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Both explicit human evaluation (3 annotators rating self-chat and human-bot dialogues) and a novel implicit human evaluation (20 annotators selecting responses from 6 deployed bots) are conducted.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "DuSincR is used as the held-out test benchmark, built by modifying the DuSinc test set, and results are reported on this set rather than training data.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Human evaluation results are broken down by chit-chat (50 opening utterances) vs. knowledge-grounded (100 opening utterances) dialogue types in Tables 2 and 3.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper briefly notes hallucination as an ongoing concern (Section 5.2) but provides no systematic discussion or examples of failure cases where the model produces incorrect or harmful outputs.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The ablation study effectively reports negative results by showing performance drops when each component is removed, confirming each component's necessity.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "GLM10B is specified with the exact parameter count, backbone paper citation (Du et al. 2022), and reference to the GitHub release; all comparison models cite specific papers.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "The actual Chinese-language prompts 𝑃𝑞, 𝑃𝑟, and 𝑃𝑘𝑟 are provided verbatim in Section 3.1-3.2 with both Chinese and English translations.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Learning rate (5×10⁻⁵ with cosine decay), batch size (256), max input length (512), auxiliary loss weight (λ=1), and knowledge pool size (m=1 at inference) are all reported.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.3 details the three-component inference pipeline: query generation module → external search engine → response generation, including time costs in Table 5.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Data preprocessing steps are documented including entity linking via HOSMEL, negative knowledge sampling using low-confidence entities, knowledge labeling strategy, and dialogue compilation from social media timing information.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The 800K online service dialogues used for training and the DuSincR test set annotations are not released as raw data files; only the code and model checkpoint are confirmed released.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data collection is described: social media from Weibo/Bilibili/Zhihu/Douban, public benchmarks listed in Table 6, and online service data from XDAI deployment Sept 1–Dec 15, 2022.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": true,
           "answer": false,
           "justification": "The paper states '3 annotators' and '20 annotators' were hired but provides no description of how they were recruited, their qualifications, or compensation.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from data collection through entity linking, knowledge labeling, knowledge pool construction, and iterative bootstrap training is documented in Sections 3.1-3.2.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The GLM10B backbone training data cutoff is not stated; the online service data cutoff (Dec 15, 2022) is mentioned but this does not address the pre-training corpus cutoff relevant to benchmark contamination.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "DuSinc data appears in both training (as benchmark data) and evaluation (DuSincR is modified DuSinc test set); this overlap is never acknowledged or addressed.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not address whether DuSinc test cases—the basis for DuSincR evaluation—may have been seen during training given DuSinc's inclusion in the training benchmark data.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": true,
           "answer": false,
           "justification": "No pre-registration is mentioned for the human evaluation studies.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": true,
           "answer": false,
           "justification": "No IRB or ethics approval is mentioned despite hiring annotators and deploying a live system with real user interactions.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": true,
           "answer": false,
           "justification": "No demographics (age, gender, language background, expertise) are reported for the annotators used in explicit or implicit evaluation.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": true,
           "answer": false,
           "justification": "No inclusion/exclusion criteria for annotators are stated beyond the implicit requirement that they participate in Chinese-language dialogue.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": true,
           "answer": true,
           "justification": "The implicit evaluation explicitly states that bot response ordering is shuffled before display to prevent position bias, as described in Section 4.3.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": true,
           "answer": true,
           "justification": "The implicit evaluation uses anonymous bots with identities not disclosed to users, preventing brand preference bias as explicitly stated in Section 4.3.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "This is a rating/selection task without longitudinal follow-up; attrition is not applicable, though the 5-turn minimum filter is mentioned.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 5 reports online time costs for each pipeline stage: query generation (1.09s), search (0.92s), response generation (1.64s), and overall latency including network (4.22s).",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Only hardware type (8×80G A100) is mentioned without total compute budget, training duration, or GPU-hours, making it impossible to assess total training cost.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "GLM-Dialog outperforms all open-source Chinese dialogue baselines on most automatic evaluation metrics on DuSincR",
       "evidence": "Table 1 shows GLM-Dialog achieves highest BLEU-4 (4.190), F1 (22.010), Rouge-L (19.464), and BertScore (0.630) among all models, though not highest on Rouge-1 and Rouge-2",
       "supported": "moderate"
     },
     {
       "claim": "Knowledge classification and iterative injection each contribute independently to dialogue quality",
       "evidence": "Table 4 ablation shows removing knowledge classification drops coherence from 1.820 to 1.730 and informativeness from 1.840 to 1.633; removing iterative injection further drops coherence to 1.757",
       "supported": "moderate"
     },
     {
       "claim": "The generated queries achieve high quality with mean cosine similarity 0.85 to ground-truth queries",
       "evidence": "Figure 4(b) shows frequency histogram of query similarity scores on 9,353 DuSinc test cases with mean 0.85; retrieved knowledge similarly scores mean 0.86",
       "supported": "moderate"
     },
     {
       "claim": "Implicit human evaluation reduces annotation bias compared to explicit rating",
       "evidence": "The paper argues that selection-based implicit rating removes explicit scoring bias but provides no empirical comparison of bias levels between methods—this is a design argument, not a measured claim",
       "supported": "weak"
     },
     {
       "claim": "GLM-Dialog achieves highest informativeness among comparison models due to external knowledge injection",
       "evidence": "Tables 2 and 3 show GLM-Dialog consistently achieves highest informativeness scores across both chit-chat and knowledge-grounded categories in self-chat and human-bot evaluation",
       "supported": "strong"
     },
     {
       "claim": "GLM-Dialog receives the most user selections in implicit evaluation among six deployed models",
       "evidence": "Figure 4(a) shows GLM-Dialog achieves highest score among models in the 10,000 total selections by 20 annotators, though exact numbers are not shown clearly",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "GLM-Dialog, a 10B-parameter Chinese knowledge-grounded dialogue model, outperforms open-source baselines (CDial-GPT, EVA2.0, PLATO-2) on most automatic and human evaluation metrics through a two-stage training strategy combining continual dialogue pre-training and noise-tolerant knowledge-infused fine-tuning. The auxiliary knowledge classification loss and iterative bootstrap injection each contribute incrementally to performance, with knowledge classification being the most critical component. An implicit human evaluation platform where annotators select from anonymous bot responses is introduced as a bias-reducing alternative to explicit multi-dimensional rating. Hallucination remains a persistent problem despite noise-tolerant training, and the model takes approximately 4 seconds total latency for online inference.",
   "red_flags": [
     {
       "flag": "Train-test overlap",
       "detail": "DuSinc benchmark data is included in training (Table 6) while DuSincR—the primary evaluation benchmark—is a modified version of the DuSinc test set. This potential contamination is never acknowledged."
     },
     {
       "flag": "No statistical tests",
       "detail": "No confidence intervals, error bars, or significance tests are reported for any of the comparative claims despite multiple model comparisons across numerous metrics."
     },
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Author Xiaohan Zhang is affiliated with Zhipu.AI, which commercializes GLM—the backbone system being evaluated favorably—but no competing interests are declared."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section; hallucination is briefly speculated about in one paragraph but no systematic limitations are discussed."
     },
     {
       "flag": "Ablation sample size",
       "detail": "Ablation studies use only 100 randomly selected conversations with no justification for this sample size and no variance reported, making it difficult to assess reliability of component-level findings."
     },
     {
       "flag": "Implicit evaluation not validated",
       "detail": "The claim that implicit evaluation reduces annotation bias is made by design argument only; no empirical comparison of bias levels between implicit and explicit methods is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "LaMDA: Language Models for Dialog Applications",
       "relevance": "Major English knowledge-grounded dialogue system used as motivation and implicit benchmark for Chinese counterpart"
     },
     {
       "title": "GODEL: Large-Scale Pre-Training for Goal-Directed Dialog",
       "relevance": "Contemporary knowledge-grounded dialogue baseline representing English dialogue LLM state-of-the-art"
     },
     {
       "title": "BlenderBot 3: A Deployed Conversational Agent that Continually Learns to Responsibly Engage",
       "relevance": "Deployed knowledge-grounded dialogue system with internet access, direct architectural parallel to GLM-Dialog"
     },
     {
       "title": "GLM: General Language Model Pretraining with Autoregressive Blank Infilling",
       "relevance": "Backbone model used for GLM-Dialog; provides the 10B parameter pre-trained checkpoint"
     },
     {
       "title": "EVA2.0: Investigating Open-Domain Chinese Dialogue Systems with Large-Scale Pre-Training",
       "relevance": "Primary Chinese dialogue baseline used for comparison"
     },
     {
       "title": "PLATO-XL: Exploring the Large-Scale Pre-Training of Dialogue Generation",
       "relevance": "Large-scale Chinese dialogue pre-training baseline"
     },
     {
       "title": "Link the World: Improving Open-domain Conversation with Dynamic Spatiotemporal-aware Knowledge (DuSinc)",
       "relevance": "Primary evaluation benchmark; DuSincR is built by extending DuSinc test set"
     },
     {
       "title": "Wizard of Wikipedia: Knowledge-Powered Conversational Agents",
       "relevance": "Foundational knowledge-grounded dialogue task formulation that this work extends to Chinese"
     },
     {
       "title": "XDAI: A Tuning-free Framework for Exploiting Pre-trained Language Models in Knowledge Grounded Dialogue Generation",
       "relevance": "Online service from which the 800K training dialogues were collected"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "System is deployed as a live WeChat chatbot with 100+ real users and an open evaluation platform, making it immediately usable by practitioners."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Results confirm expected patterns—larger models with knowledge grounding outperform smaller baselines—with no surprising or counterintuitive findings."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper briefly notes hallucination as a limitation but raises no substantive AI safety concerns."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Standard system paper with no controversy; the implicit evaluation novelty is presented modestly."
     },
     "demo_ability": {
       "score": 3,
       "justification": "A live WeChat demo and online evaluation platform are both publicly deployed and directly usable."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Tsinghua University and the GLM/ChatGLM lineage are well-known in the Chinese AI research community; Zhipu.AI has commercial visibility."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }