ai-research-survey

scan.json (27084B)

---

 {
   "paper": {
     "title": "Bottom-up Domain-specific Superintelligence: A Reliable Knowledge Graph is What We Need",
     "authors": [
       "Bhishma Dedhia",
       "Yuval Kansal",
       "Niraj K. Jha"
     ],
     "year": 2025,
     "venue": "Preprint (arXiv)",
     "arxiv_id": "2507.13966"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a project website URL (https://kg-bottom-up-superintelligence.github.io/) on the title page. While the paper itself does not contain a direct GitHub link in the body text, this project page serves as the artifact portal. The paper does not explicitly state code is released but provides this URL as the project resource."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes curating 24,000 QA tasks and the ICD-Bench evaluation suite (3,675 items), but does not provide a download link for the dataset. The UMLS KG used is a publicly available resource, but the curated curriculum and ICD-Bench data are not explicitly released with a URL."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C describes the SFT setup (8×H100 GPUs, LoRA rank=16, learning rate schedule, optimizer settings) but does not provide a requirements.txt, Dockerfile, or detailed library version specifications needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the pipeline methodology in detail (Sections 3.1, 3.2, Algorithm 1 in Appendix A.3) and the SFT setup (Appendix C), but does not provide step-by-step reproduction instructions, README with commands, or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 'Bootstrapped confidence intervals over 500 samples were < 0.75%' for ICD-Bench results (Section 6.1, Fig. 6 caption, Fig. 7 caption)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims curriculum-tuned models 'significantly outperform' baselines across multiple sections (Sections 6.1, 6.2), but no formal statistical significance tests (p-values, t-tests, etc.) are reported. Claims of difference are based on comparing accuracy numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context, e.g., 'outperform all open-source baselines across inference budgets by 10-20%' (Section 6.2, O2.1), and provides baseline context in figures and tables (Table 1, Fig. 7) showing absolute accuracy values and improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The training dataset size (24,000 tasks) and ICD-Bench size (3,675 items) are stated but not justified with a power analysis or explanation of why these specific sizes were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports bootstrapped confidence intervals for ICD-Bench results but does not report variance or standard deviation across experimental runs. The SFT models appear to be trained once (single run), and no multi-seed results are presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 compares against four baselines: QwQ-32B base model, DeepSeek-R1-Distilled Qwen, o3, and Gemini-2.5-Pro. Table 1 adds comparisons against MedGemma, Meerkat, Qwen3, and Sky-T1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include o3 (OpenAI, 2025), Gemini-2.5-Pro (Google, 2025), DeepSeek-R1 (2025), Qwen3 (2025), and Meerkat (2025). These are contemporary state-of-the-art reasoning models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 presents detailed ablations disentangling the effects of curriculum depth (1-hop vs. 3-hop), complexity sampling, and diversity sampling on model performance. Three model variants (QwQ-Med-1, -2, -3) also serve as ablations of increasing curriculum complexity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses accuracy across 15 ICD-Bench categories, per-difficulty-bin accuracy (Section 6.3), recall vs. reasoning accuracy analysis (Section 6.5), and evaluation on 4 external benchmarks (MedQA, PubMedQA, MedMCQA, MMLU-Med in Table 1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is conducted. All evaluation is automated via multiple-choice accuracy. For a paper claiming 'superintelligence' in medical reasoning, human expert evaluation of the model's reasoning traces would strengthen the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "ICD-Bench is explicitly separated from training data via decontamination (Section 5.1): exact KG path overlap is excluded and an 18-gram text overlap filter is applied. ICD-Bench uses 4- and 5-hop paths while training uses 1-3 hop paths. External benchmarks (MedQA, etc.) are independent."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across all 15 ICD-Bench categories (Fig. 1, Fig. 7), across 5 difficulty bins (Fig. 8), and across hop lengths (Fig. 10). This is a strength of the evaluation design."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 discusses failure modes: 'even curriculum-tuned models struggle to reason correctly due to insufficient recall' in certain categories like Drugs and Mediators. Appendix D provides qualitative examples comparing base model failures (Examples 5-7) with detailed analysis of reasoning-recall gaps."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 reports that 'using only three-hop chains slightly degrades performance' on easy tasks (O4.2), and Section 6.6 notes QwQ-Med-3 scores lower than the base QwQ on MedQA (82.72 vs. 85.62), acknowledging the trade-off."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims QwQ-Med-3 'significantly outperforms state-of-the-art open-source and proprietary reasoning models on all categories of ICD-Bench,' which is supported by Fig. 1, Fig. 7, and Section 6.2. The claim about transfer to external benchmarks is supported by Table 1. The claim about performance gap widening on hardest tasks is supported by Fig. 8."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about curriculum tuning improving performance, which are supported by controlled ablation studies (Section 6.4) with single-variable manipulations (depth, complexity sampling, diversity sampling) under fixed FLOPs budgets. The three progressive models (Med-1, -2, -3) provide controlled comparisons."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses the term 'superintelligence' in the title and throughout, but results are limited to multiple-choice medical QA tasks derived from a single KG (UMLS). The Limitations section (Section 8) acknowledges constraints ('generalizability to other domains... remains to be fully validated' and 'we limit our focus to generating closed-ended multiple choice question tasks'), but the title 'Bottom-up Domain-specific Superintelligence' significantly overstates what closed-ended MCQ performance demonstrates."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the performance gains. For instance, it does not consider whether the gains could be attributed to data augmentation effects, exposure to medical terminology, or the specific characteristics of MCQ format rather than 'domain-specific superintelligence.' The Limitations section discusses scope constraints but not alternative causal explanations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Gemini 2.0 Flash' (Section 3.1.1), 'Gemini-2.5-Pro' (Section 3.1.2), 'QwQ-32B' (Section 5.2), 'Qwen 2.5-72B' (Section 3.1.2), 'o3' (Section 6.2), but none include specific API versions, snapshot dates, or model card version identifiers. These are marketing names without version specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides actual prompt templates in Appendix A.2 (Prompt 1 for QA generation), Appendix A.3 (Prompt 2 for thinking trace generation, Prompt 3 for correctness filtering), and Appendix F (Prompt 4 for benchmark evaluation). These contain the full text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C reports LoRA rank=16, α=16, batch size=16, gradient accumulation every 2 steps, cosine learning rate schedule with peak 1e-5, 5% warmup, Adam with β1=0.9, β2=0.95, weight decay=1e-4, 8 epochs. Inference: temperature=0.6, K values for parallel scaling. Table 3 in Appendix F reports inference hyperparameters for all models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The approach does not use agentic scaffolding. It is a fine-tuning and inference pipeline without tool use, retry logic, or agent-based workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.2 and Algorithm 1 describe the full data preprocessing pipeline: diversity sampling, complexity sampling, quality filtering (API failures, formatting, distractor quality), thinking trace generation, and dual-model correctness filtering. Section 5.1 describes the decontamination steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Discussion) contains a dedicated 'Limitations' paragraph that discusses five specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations paragraph in Section 8 identifies specific threats: (1) closed vocabulary of the KG constraining conceptual coverage, (2) limitation to closed-ended MCQ tasks, (3) difficulty heuristic relying on oracle answers, (4) efficacy demonstrated only in medicine where reliable KG exists, and (5) generalizability to domains lacking canonical KGs not validated."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Limitations section acknowledges several constraints, the paper does not explicitly state what the results do NOT show. The paper claims 'superintelligence' but does not explicitly bound this claim to MCQ performance on KG-derived tasks. The title and framing significantly exceed what the evidence demonstrates, and no explicit 'what we do not claim' statement is provided."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The curated 24,000 QA training tasks, ICD-Bench data, and generated thinking traces are not made available for independent verification. Only aggregate results are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Algorithm 1 describe the data collection procedure in detail: path traversal on the UMLS KG, QA generation via LLM, quality filtering, thinking trace generation, and correctness filtering with dual LLM verification."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. Data is generated synthetically from a knowledge graph using LLMs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline is documented in Section 3.1.2 (5 steps), Algorithm 1 (Appendix A.3), and Section 5.1 (decontamination). Figure 3 provides a visual overview. However, exact counts of items filtered at each stage are not provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The 'Acknowledgments and Disclosure of Funding' section states: 'This work was supported by NSF under Grant No. CNS-2216746' and mentions computational resources from Princeton Research Computing and the Princeton Language and Intelligence Initiative."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with the Department of Electrical and Computer Engineering at Princeton University, clearly stated on the title page. No evaluated products are from Princeton."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is NSF (National Science Foundation), a government agency with no financial stake in the outcome of the research. The computational resources are from Princeton University."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper. The absence of a disclosure statement means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses QwQ-32B as the base model and evaluates against models like o3, Gemini-2.5-Pro, and DeepSeek-R1, but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes a two-fold decontamination procedure: (1) excluding QA pairs whose KG paths exactly match ICD-Bench paths, and (2) an 18-gram text overlap filter. This addresses train/test overlap between the curated curriculum and ICD-Bench."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "For external benchmarks (MedQA, MedMCQA, MMLU-Med, PubMedQA), no contamination analysis is provided. The base QwQ-32B model was trained on web-scale data that could include these benchmark questions. The paper does not discuss whether the base model or the curriculum-tuned model may have seen these external benchmark questions during pre-training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports average thinking tokens per question across scaling configurations (Fig. 6 x-axis, Fig. 7 x-axis), enabling assessment of inference cost. Section 6.1 systematically varies inference budgets with K values and reports accuracy vs. token consumption trade-offs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 states 'fine-tuned using LoRA with rank 16 on 8 H100 NVIDIA GPUs, with each run taking approximately 20 hours.' The paper also notes curriculum generation is a 'one-time cost' (Section 3.1) though does not quantify it."
       }
     }
   },
   "claims": [
     {
       "claim": "QwQ-Med-3 significantly outperforms state-of-the-art open-source and proprietary reasoning models on all 15 categories of ICD-Bench.",
       "evidence": "Fig. 1, Fig. 7, and Section 6.2 show QwQ-Med-3 outperforms QwQ-32B base, DeepSeek-R1-Distilled Qwen, o3, and Gemini-2.5-Pro across all ICD-Bench categories, with 10-20% improvement over open-source baselines.",
       "supported": "moderate"
     },
     {
       "claim": "Performance improves with deeper and more diverse KG curricula, with curriculum depth proving especially crucial for the most challenging reasoning tasks.",
       "evidence": "Section 6.4 and Fig. 9 show ablations across 1-hop only, 3-hop only, balanced, and full 24K datasets. The hardest tasks benefit most from 3-hop paths, while easy tasks benefit more from diversity.",
       "supported": "strong"
     },
     {
       "claim": "Curriculum-tuned models can transfer acquired KG primitives to external medical QA benchmarks beyond the original KG.",
       "evidence": "Table 1 shows QwQ-Med-3/parallel-scaling achieves competitive or improved results on MedQA (85.39), PubMedQA (78.19), MedMCQA (73.25), and MMLU-Med (92.90) versus baselines. However, on MedQA the base QwQ (85.62/87.09) outperforms QwQ-Med-3 (82.72/85.39).",
       "supported": "moderate"
     },
     {
       "claim": "Curriculum-tuned models show greater marginal gains on harder tasks, with the base model offering near-zero accuracy on the hardest tasks.",
       "evidence": "Section 6.3 and Fig. 8 show difficulty-stratified results where the base model drops to near-zero on the hardest bin while QwQ-Med-3 maintains meaningful accuracy. The performance gap widens with difficulty.",
       "supported": "strong"
     },
     {
       "claim": "The approach demonstrates domain-specific superintelligence, taking a step towards medical superintelligence.",
       "evidence": "The paper demonstrates strong MCQ performance on ICD-Bench and external benchmarks but does not demonstrate reasoning beyond closed-ended multiple-choice format. No open-ended clinical reasoning, real-world validation, or comparison to human expert performance is provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper proposes a task-generation pipeline that traverses knowledge graph paths to create grounded medical reasoning tasks, then fine-tunes QwQ-32B on 24,000 such tasks to produce QwQ-Med-3. On the newly introduced ICD-Bench (3,675 medical QA items across 15 specialties), QwQ-Med-3 outperforms o3, Gemini-2.5-Pro, and other baselines across all categories. Ablation studies show curriculum depth is most critical for hard tasks while diversity helps across all difficulties. The curriculum-tuned model shows competitive transfer to external medical benchmarks (MedQA, PubMedQA, MedMCQA, MMLU-Med), though it underperforms the base model on MedQA.",
   "red_flags": [
     {
       "flag": "Overclaimed title and framing",
       "detail": "The paper claims 'domain-specific superintelligence' and 'a step towards medical superintelligence' based on multiple-choice QA performance. Superintelligence implies surpassing human expert capability across the domain, but the evaluation is limited to closed-ended MCQ tasks generated from a single KG. No comparison to human expert performance is provided, and no open-ended clinical reasoning is evaluated."
     },
     {
       "flag": "Self-constructed benchmark advantage",
       "detail": "ICD-Bench is constructed by the same team using the same KG (UMLS) and similar pipeline as the training data. While decontamination is performed (path exclusion + 18-gram filter), the benchmark inherently tests the same type of KG-grounded reasoning the model was trained on, which may overstate generalization. The model's advantage on ICD-Bench vs. external benchmarks supports this concern."
     },
     {
       "flag": "No formal significance testing",
       "detail": "The paper repeatedly uses the word 'significantly' to describe performance differences but provides no formal statistical significance tests. Bootstrapped CIs are reported as <0.75% but no comparison tests between models are conducted."
     },
     {
       "flag": "Single training run",
       "detail": "Each model variant appears to be trained exactly once. No multi-seed experiments are reported, making it impossible to assess the stability of results across random initialization or data sampling."
     },
     {
       "flag": "Base model outperforms on external benchmark",
       "detail": "On MedQA (Table 1), the base QwQ model (85.62/87.09 with scaling) outperforms QwQ-Med-3 (82.72/85.39 with scaling), suggesting the curriculum tuning may cause some forgetting on standard medical tasks even while improving KG-grounded reasoning."
     }
   ],
   "cited_papers": [
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Major open-source reasoning model used as a baseline, representing the RL-based approach to reasoning in LLMs."
     },
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Foundational work on inference-time scaling that this paper builds upon for its parallel and iterative refinement strategies."
     },
     {
       "title": "s1: Simple Test-Time Scaling",
       "authors": ["Niklas Muennighoff", "Zitong Yang", "Weijia Shi"],
       "year": 2025,
       "arxiv_id": "2501.19393",
       "relevance": "Inference-time scaling approach used as basis for the iterative refinement strategy in this paper."
     },
     {
       "title": "Emergent Abilities of Large Language Models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani"],
       "year": 2022,
       "arxiv_id": "2206.07682",
       "relevance": "Foundational work on emergent abilities in LLMs, relevant to claims about emergence of domain-specific reasoning."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "arxiv_id": "2304.15004",
       "relevance": "Challenges the notion of emergent abilities in LLMs, directly relevant to evaluating this paper's claims about 'emergence' of domain-specific reasoning."
     },
     {
       "title": "QA-GNN: Reasoning with Language Models and Knowledge Graphs for Question Answering",
       "authors": ["Michihiro Yasunaga", "Hongyu Ren", "Antoine Bosselut", "Percy Liang", "Jure Leskovec"],
       "year": 2021,
       "relevance": "Key prior work on integrating knowledge graphs with language models for QA, which this paper builds upon for the UMLS KG construction."
     },
     {
       "title": "Small Language Models Learn Enhanced Reasoning Skills from Medical Textbooks",
       "authors": ["Hyunjae Kim"],
       "year": 2025,
       "relevance": "Meerkat model family used as a baseline; demonstrates potential of small LMs with careful data curation for medical reasoning."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational work on chain-of-thought reasoning that underpins the thinking trace approach used in this paper."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["Edward J. Hu", "Yelong Shen", "Phillip Wallis"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "The fine-tuning method used in this paper for curriculum-tuning the QwQ-32B model."
     },
     {
       "title": "Capabilities of GPT-4 on Medical Challenge Problems",
       "authors": ["Harsha Nori", "Nicholas King", "Scott Mayer Mckinney"],
       "year": 2023,
       "arxiv_id": "2303.13375",
       "relevance": "Demonstrates LLM capabilities on medical exams, providing context for evaluating the medical QA benchmark results in this paper."
     },
     {
       "title": "Curriculum Learning",
       "authors": ["Yoshua Bengio", "Jerome Louradour", "Ronan Collobert", "Jason Weston"],
       "year": 2009,
       "relevance": "Foundational work on curriculum learning that motivates the progressive training approach used in this paper."
     },
     {
       "title": "AlphaEvolve: A Coding Agent for Scientific and Algorithmic Discovery",
       "authors": ["Alexander Novikov"],
       "year": 2025,
       "arxiv_id": "2506.13131",
       "relevance": "Example of domain-specific AI achieving expert-level performance, relevant to the broader discussion of domain-specific superintelligence."
     }
   ]
 }