ai-research-survey

scan.json (29812B)

---

 {
   "paper": {
     "title": "Mix-of-Granularity: Optimize the Chunking Granularity for Retrieval-Augmented Generation",
     "authors": [
       "Zijie Zhong",
       "Hanwen Liu",
       "Xiaoya Cui",
       "Xiaofan Zhang",
       "Zengchang Qin"
     ],
     "year": 2024,
     "venue": "International Conference on Computational Linguistics",
     "arxiv_id": "2406.00456",
     "doi": "10.48550/arXiv.2406.00456"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MoG dynamically selects optimal chunking granularity for RAG retrieval via a trained router, improving accuracy over fixed-granularity MedRAG baselines across five medical QA benchmarks and five backbone LLMs (average 5% over MedRAG on smaller models). MoGG extends MoG by reorganizing documents as graphs, enabling retrieval of distantly situated snippets with even fewer training samples. Performance gains are larger on weaker LLMs (GLM3, Qwen1.5) and MoG does not consistently outperform chain-of-thought prompting without retrieval, which the authors attribute to noise injection from unfiltered retrieved snippets.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'The code of both MoG and MoGG are released in https://github.com/ZGChung/Mix-of-Granularity.' A specific GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All corpora (PubMed, StatPearls, Textbooks, Wikipedia) and QA datasets (MMLU-Med, MedQA-US, MedMCQA, PubMedQA*, BioASQ-Y/N) are publicly available standard benchmarks described in Section 4.1 and Appendix B."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions PyTorch framework and Nvidia GeForce 3090/4090 GPUs (Section 4.2), but provides no requirements.txt, Dockerfile, or detailed library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The experimental setup (Section 4.2) describes parameters but not a reproducible workflow."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report ± notation for all results (e.g., '0.5198±0.04'). These appear to be confidence intervals based on dataset size rather than variance across runs, but CI notation is present."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. The paper claims MoG 'consistently enhanced the performance' (Section 4.3) based solely on comparing accuracy numbers without any p-values, t-tests, or other significance testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 states 'MoG shows an average improvement of 5% compared to MedRAG and 8.7% compared to CoT' with baseline context. Tables provide both baseline and improved numbers enabling effect size calculation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes or power analysis. The paper uses existing benchmark datasets (500-4183 questions per dataset) without discussing whether these sizes are adequate for the claimed comparisons."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The ± values in tables are constant per dataset column across all methods (e.g., MMLU always ±0.03 or ±0.04), indicating they are binomial proportion confidence intervals based on dataset size, not variance across experimental runs. No multi-run variance, standard deviation, or seed-based variability is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are included: CoT (chain-of-thought prompting without retrieval) and MedRAG (fixed single-granularity RAG system from Xiong et al., 2024), described in Section 4.3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MedRAG (Xiong et al., 2024) is a contemporary baseline from the same year. The MIRAGE benchmark framework is also current."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 (Appendix K) presents an ablation removing the router from MoGG. Appendix E compares different soft label construction methods (TF-IDF, RoBERTa, hitrate). Appendix F varies the number of candidate snippets."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only exact matching accuracy is used as the evaluation metric across all experiments. While results are broken down by dataset, the metric is always the same."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is performed. Appendix I mentions manual verification of ~10% of degraded samples to diagnose causes, but this is error analysis, not human evaluation of output quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets are used. For MedMCQA, they specifically use the 'dev' set (Appendix B.3). The other datasets use their standard evaluation splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (5 QA benchmarks), by backbone LLM (5 models), by training corpus (4 corpora in Table 7), and by number of candidate snippets (Appendix F)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I provides detailed analysis of degraded samples. The authors manually verified ~10% of degraded cases and found 95% were caused by noise injection from retrieved snippets, not MoG design flaws."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that MoG does not consistently beat CoT ('though not necessarily better than CoT', Section 4.3), and that noise from retrieved snippets can degrade performance. MoGG also does not consistently outperform MoG across all settings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims MoG/MoGG 'effectively predict optimal granularity levels, significantly enhancing the performance of the RAG system.' Tables 1 and 2 show improvements over MedRAG baselines in average accuracy across most configurations."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study in Table 9 removes the router component from MoGG, showing it degrades to MedRAG-level performance. This controlled single-variable manipulation provides adequate evidence for the causal claim that the router mechanism drives the improvement."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to 'Optimize the Chunking Granularity for Retrieval-Augmented Generation' generally, but all experiments are on medical QA datasets only. Section 4.1 further states 'We posit that significant improvements demonstrated by the tests on this knowledge-intensive field suggest MoG's potential effectiveness in other domains' — an unbounded generalization claim."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses noise as a cause of degradation (Appendix I) but does not consider alternative explanations for the observed improvements — e.g., whether improvements stem from ensemble effects of multi-granularity retrieval rather than optimal granularity selection, or whether the soft label training signal is confounded with corpus characteristics."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures exact matching accuracy on multiple-choice QA tasks and frames claims at this same granularity. No proxy gap exists between measurement and claim."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 (Appendix C) lists exact model versions: 'gpt-3.5-turbo-16k', 'Meta-Llama-3-8B', 'internlm2-123b', 'chatglm3-6b', 'Qwen1.5-MoE-A2.7B' with URLs to specific model pages."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. The paper states snippets are 'injected into the backbone LLM via prompt' (Section 3.1) but never shows the actual prompt template or text used for the QA task."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports: Adam optimizer, learning rate 0.001, top-3 retrieval per granularity, top-2 final snippet selection, 1000 epochs per training session, ~12GB GPU memory for training, 6GB for inference."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. MoG is a retrieval optimization method with a trained router, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.1 describes the chunking procedure (non-overlapping, each coarser level = 2× previous, 5 levels). Section 4.1 states chunking sizes relative to MedRAG ({1/2, 1, 2, 4, 8}×). Section 3.3 describes graph construction for MoGG. Soft label construction is detailed in Section 3.2.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Limitations and Broader Impacts' provides a dedicated discussion of three specific limitations and a security concern."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 lists specific limitations: (1) granularity levels are manually assigned requiring grid search, (2) the router uses only semantic query information without query type or expected response length, (3) lack of length normalization in retrieval scoring, (4) security risk from a compromised router."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Instead, it generalizes from medical QA to other domains: 'We posit that significant improvements demonstrated by the tests on this knowledge-intensive field suggest MoG's potential effectiveness in other domains with lower knowledge dependencies and higher error tolerances' (Section 4.1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model predictions, retrieved snippets, router weights per query) is released. Only aggregated accuracy numbers are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Appendix B describe all corpora and QA datasets with statistics (Table 3): document counts, snippet counts, average lengths, domains, dataset sizes, and number of options."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks and corpora."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: corpora → chunking at 5 granularity levels → BM25 indexing → soft label construction → router training → inference. For MoGG, the graph construction step is additionally described (Section 3.3). Corpus statistics are provided in Table 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgements thank 'supervisors and colleagues at Shanghai Artificial Intelligence Laboratory' and reviewers, but no explicit funding sources, grants, or sponsors are listed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed on the first page: Shanghai AI Laboratory, Beihang University, and theSight Technology."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot be determined since funding is not disclosed. The work is affiliated with Shanghai AI Laboratory which may have interests in RAG system improvements."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five backbone LLMs (GPT-3.5, InternLM2, Llama3, GLM3, Qwen1.5) despite evaluating them on public benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the QA benchmark questions (MMLU, MedQA, etc.) appeared in the training data of the backbone LLMs. These are well-known public benchmarks likely present in training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU (2021), MedQA (2018), MedMCQA (2022), PubMedQA (2019), and BioASQ (2015-2023) were all publicly available before the training cutoffs of the tested LLMs. No contamination discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix M (Table 11) reports wall-clock execution times for different numbers of granularity levels. Section 4.5 states router requires ~12GB GPU memory for training and 6GB for inference."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: '35 training sessions and over a hundred inferences, each training session taking around 4 hours for 1000 epochs.' GPU types (3090, 4090) and memory requirements are specified. Storage overhead is 2.7× corpus size."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from single runs, with ± values being dataset-size-based confidence intervals rather than seed-based variance."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. 'Each training job is run until the convergence of the loss value' (Section 4.2) suggests single runs per configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The learning rate (0.001), soft label values (0.8/0.2), kr values, and kgraph are presented without documenting how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Key design choices like soft label values '[0.8, 0.2, 0] or [0.7, 0.3, 0] yields similar results' (Section 3.2.2) are mentioned but the selection process and number of alternatives tested is not systematically documented."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs comparisons across 5 LLMs × 5 datasets × multiple methods with no correction for multiple comparisons applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own MoG/MoGG system against their own implementation of baselines (MedRAG, CoT) without acknowledging or mitigating author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Appendix M (Table 11) reports execution time across different numbers of granularity levels, and Section 4.5 discusses the marginal cost increase with more granularity levels relative to the performance gains."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the MIRAGE benchmark QA tasks actually measure the claimed retrieval quality improvements, or whether exact matching accuracy is an adequate construct for evaluating chunking optimization."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. MoG is a retrieval optimization method, not an agent-based system."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LLM training data includes information from after the benchmark creation dates. All five benchmarks predate the tested LLMs' training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states: 'To prevent knowledge leakage, only the question is used (options excluded) to retrieve reference documents from the external knowledge database.' This explicitly addresses one form of feature leakage in the retrieval setup."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test examples in the QA datasets share structural similarities or whether the soft-label training data overlaps with evaluation data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap analysis) is applied. The option exclusion is a design choice, not a detection method for training data contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "MoG consistently enhances RAG system performance over MedRAG baseline across different backbone LLMs",
       "evidence": "Table 1 shows MoG outperforms MedRAG on average accuracy for all 5 backbone LLMs when trained with MedCorp corpus (GLM3: 0.4923 vs 0.4804, GPT-3.5: 0.6631 vs 0.6484, InternLM: 0.6057 vs 0.5428, Llama3: 0.6648 vs 0.5900, Qwen1.5: 0.5174 vs 0.4558).",
       "supported": "moderate"
     },
     {
       "claim": "MoG improves accuracy more on weaker LLMs than stronger ones",
       "evidence": "Section 4.3 notes this pattern. From Table 1: Qwen1.5 gains +6.2pp, GLM3 gains +1.2pp over MedRAG averages, while GPT-3.5 gains +1.5pp. The pattern is directionally consistent but not universal (InternLM gains +6.3pp despite being larger).",
       "supported": "weak"
     },
     {
       "claim": "MoGG brings more significant improvement over MedRAG than MoG when trained on smaller corpora",
       "evidence": "Section 4.4 claims this by comparing Tables 1 and 2, noting MoGG trained on Textbooks (0.2% of MedCorp) still yields improvements. However, MoGG does not consistently outperform MoG across all LLM-dataset combinations in Table 2.",
       "supported": "weak"
     },
     {
       "claim": "The router mechanism is the key component driving MoGG's improvement",
       "evidence": "Table 9 (Appendix K) shows ablation: MoGG without router (0.5120 avg) performs similarly to MedRAG (0.5127 avg), while MoGG with router achieves 0.6262 avg on Llama3.",
       "supported": "strong"
     },
     {
       "claim": "Increasing granularity levels adds only marginal execution time",
       "evidence": "Table 11 (Appendix M) shows global average time increases from 13.16s (1 level with router) to 17.93s (5 levels), with the bottleneck being LLM API call time.",
       "supported": "strong"
     },
     {
       "claim": "Performance degradation from MoG is caused by noise in retrieved snippets, not the routing mechanism",
       "evidence": "Appendix I reports manual verification of ~10% of degraded samples, finding in 95% of cases the snippets were correctly retrieved but the LLM changed its answer based on noisy content.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All performance comparisons rely on comparing point estimates. The ± values in tables are binomial CIs based on dataset size (they are constant per dataset across methods), not variance across experimental runs. No significance tests are applied to any comparison despite claiming 'statistically and practically significant' improvements."
     },
     {
       "flag": "Single-domain evaluation with general claims",
       "detail": "All experiments use medical QA datasets only, yet the paper title and claims frame the contribution as generally applicable to RAG systems. The paper explicitly extrapolates: 'We posit that significant improvements... suggest MoG's potential effectiveness in other domains.'"
     },
     {
       "flag": "No benchmark contamination analysis",
       "detail": "Five public benchmarks (MMLU 2021, MedQA 2018, PubMedQA 2019, MedMCQA 2022, BioASQ 2015-2023) are used with LLMs that likely saw them during training. The paper's main contribution is a retrieval method, but contaminated baseline LLM knowledge confounds the comparison between RAG-enhanced and non-RAG (CoT) conditions."
     },
     {
       "flag": "Inconsistent results presented as consistent improvement",
       "detail": "MoG does not beat CoT on several LLM-dataset combinations (acknowledged in Section 4.3), and MoGG does not consistently beat MoG. The abstract's claim of 'significantly enhancing the performance' overstates the evidence."
     },
     {
       "flag": "Single-run results",
       "detail": "No evidence of multiple experimental runs. Training converges once per configuration, and the ± values are not run-based variance. Result stability across seeds or initializations is unknown."
     }
   ],
   "cited_papers": [
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper defining the retrieval-augmented generation paradigm for enhancing LLMs with external knowledge."
     },
     {
       "title": "Benchmarking retrieval-augmented generation for medicine",
       "authors": ["Guangzhi Xiong", "Qiao Jin", "Zhiyong Lu", "Aidong Zhang"],
       "year": 2024,
       "arxiv_id": "2402.13178",
       "relevance": "MIRAGE benchmark used as the primary evaluation framework; MedRAG serves as the main baseline for comparison."
     },
     {
       "title": "RAPTOR: Recursive abstractive processing for tree-organized retrieval",
       "authors": ["Parth Sarthi", "Salman Abdullah", "Aditi Tuli"],
       "year": 2024,
       "arxiv_id": "2401.18059",
       "relevance": "Hierarchical retrieval method organizing snippets as trees with multi-granularity access, directly related to granularity optimization in RAG."
     },
     {
       "title": "Retrieval-augmented generation for large language models: A survey",
       "authors": ["Yunfan Gao", "Yun Xiong", "Xinyu Gao"],
       "year": 2024,
       "arxiv_id": "2312.10997",
       "relevance": "Comprehensive RAG survey covering retrieval strategies, graph indexing, and generation techniques relevant to the survey scope."
     },
     {
       "title": "Active retrieval augmented generation",
       "authors": ["Zhengbao Jiang", "Frank F. Xu", "Luyu Gao"],
       "year": 2023,
       "arxiv_id": "2305.06983",
       "relevance": "Adaptive retrieval strategy that dynamically decides when to retrieve, relevant to dynamic RAG optimization research."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "CoT prompting technique used as a baseline comparison across all experiments in this paper."
     },
     {
       "title": "Demonstrate-search-predict: Composing retrieval and language models for knowledge-intensive NLP",
       "authors": ["Omar Khattab", "Keshav Santhanam", "Xiang Lisa Li"],
       "year": 2023,
       "arxiv_id": "2212.14024",
       "relevance": "DSP framework for composing retrieval and generation, representing information fusion techniques for RAG systems."
     },
     {
       "title": "Improving language models by retrieving from trillions of tokens",
       "authors": ["Sebastian Borgeaud", "Arthur Mensch", "Jordan Hoffmann"],
       "year": 2022,
       "arxiv_id": "2112.04426",
       "relevance": "RETRO architecture integrating retrieval at intermediate layers of LLMs, a key related work on retrieval-augmented language modeling."
     },
     {
       "title": "Atlas: Few-shot learning with retrieval augmented language models",
       "authors": ["Gautier Izacard", "Patrick Lewis", "Maria Lomeli"],
       "year": 2022,
       "arxiv_id": "2208.03299",
       "relevance": "Multiple retrieval strategy for RAG systems relevant to the multi-granularity retrieval approach proposed in MoG."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing in-context learning capabilities that underpin RAG-based approaches to knowledge augmentation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "RAG chunking optimization is directly relevant to practitioners building retrieval systems, with released code, though limited to medical domain validation."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Dynamic chunking is a reasonable extension of existing RAG practices rather than a surprising finding; the MoE-inspired routing is a natural architectural choice."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised as findings; the router security mention in limitations is speculative, not a demonstrated risk."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with established methods or labs; the paper presents incremental improvements over existing baselines."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is released on GitHub (https://github.com/ZGChung/Mix-of-Granularity), enabling practitioners to try the method."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Shanghai AI Laboratory has moderate recognition in the Chinese AI research community but is not a top-tier globally recognized lab."
     }
   }
 }