ai-research-survey

scan.json (22819B)

---

 {
   "paper": {
     "title": "CORE: Comprehensive Ontological Relation Evaluation for Large Language Models",
     "authors": ["Satyam Dwivedi", "Sanjukta Ghosh", "Shivam Dwivedi", "Nishi Kumari", "Anil Thakur", "Anurag Purushottam", "Deepak Alok", "Praveen Gatla", "Manjuprasad B", "Bipasha Patgiri"],
     "year": 2026,
     "venue": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 states 'data and code are hosted at Hugging Face and GitHub' and references core.vaikhari.ai."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the benchmark and dataset are available at core.vaikhari.ai with data hosted on Hugging Face."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are provided. The paper mentions 'identical hardware specifications' for open-source models but does not specify them."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The prompt is given in Appendix A but no instructions for running the evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '48.25–70.9% overall accuracy') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims systematic differences between related and unrelated pair performance but uses no statistical significance tests to support these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context: accuracy on related pairs (86.5–100%) vs unrelated pairs (0–41.35%), ECE increase of 2–4x on unrelated pairs, and semantic collapse rate of 37.6%. These provide magnitude with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 203 questions (102 open, 101 blind). No justification is given for why this sample size is sufficient, nor is any power analysis provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Each model appears to have been run once with deterministic settings, with no repeated runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "A human baseline from 1,000+ participants is included (Table 1), achieving 92.6% overall accuracy. 29 LLMs are compared against each other and against this human baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 29 models include contemporary frontier models with cutoff date January 22, 2026, including GPT-5.2, Claude-Opus-4.5, Gemini-3-flash, Llama-4-scout, etc."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: accuracy, balanced accuracy, ECE, overconfident error rate, and semantic collapse rate (Section 4.3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human baseline evaluation from 1,000+ participants is included, establishing human performance on the benchmark (Section 3.4, Table 1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark includes a blind subset of 101 questions 'withheld for internal analysis and validation' (Section 3.1), separate from the 102 open questions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by related vs. unrelated pairs (Tables 3, 4), by difficulty level (Table 5), and per-relation performance is discussed in Section 5.4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 provides a qualitative failure example ('Hospital is to flying as wolf is to_?') showing how models construct spurious relational narratives. Section 6.3 discusses confidence-coherence misalignment."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper reports a negative result: LLMs systematically fail on unrelated pairs (0–41.35% accuracy). Section 5.5 reports further degradation to ~2% on the 225K dataset."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (48.25–70.9% accuracy, 0–41.35% on unrelated, 37.6% semantic collapse rate, ~2% on 225K dataset) are supported by results in Sections 5.1–5.5 and Tables 3–5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.2 makes causal claims about architectural and training objective biases causing failures ('The observed failures likely arise from the interaction between model architecture, training objectives, and evaluation formulation'). These are speculative explanations without controlled experiments to support them."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Limitations) explicitly bounds scope: English-only, multiple-choice format only, text-only evaluation. Claims are hedged appropriately (e.g., 'suggests' rather than definitive statements)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses multiple contributing factors (architecture, training objectives, evaluation formulation). Section 6.3 discusses confidence-coherence misalignment as an alternative interpretation. The paper acknowledges it cannot establish definitive causal mechanisms."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 lists models by marketing names (e.g., 'Claude-Opus-4.5', 'GPT-5.2', 'Gemini-3-flash') without specific API versions or snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides the full CORE_PROMPT template used for model inference, including the complete JSON response format and field specifications. The placeholders {question_text} and {options_block} are filled from the benchmark questions which are also released."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'deterministic inference with recommended inference settings' for proprietary models and 'default configurations' for open-source models, but does not specify temperature, top-p, or other specific hyperparameter values."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated with single-turn prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 describes the validation pipeline: 250 initial questions → three-pass expert review → 203 questions with perfect inter-annotator agreement (Cohen's κ = 1.0). The question design and relation type selection process is documented in Sections 3.1–3.3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' provides a dedicated limitations discussion covering language scope, format limitations, and text-only evaluation constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: English-only evaluation may reflect language-specific properties, multiple-choice format may scaffold performance differently than open-ended generation, and text-only evaluation excludes multimodal reasoning."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states what was NOT tested: non-English languages, open-ended generation, multimodal settings. These are specific exclusions rather than generic disclaimers."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data and 225K dataset are stated to be available on Hugging Face and at core.vaikhari.ai."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1–3.4 describe the dataset construction: 225K MCQs spanning 74 disciplines, 203-question benchmark subset, question format, relation type selection based on ontological frameworks, and expert validation process."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The human baseline used 'over 1,000 participants in India, spanning undergraduate to postdoctoral education levels' but does not describe how participants were recruited, through what channels, or whether recruitment could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from 250 initial questions through three-pass expert review to 203 final questions with Cohen's κ = 1.0 is documented in Section 3.4. The benchmark split into open (102) and blind (101) subsets is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed anywhere in the paper. The Acknowledgments section thanks participants and academicians but does not mention funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Vaikhari AI, IIT BHU, IIT Delhi, BHU, GSSSIETW Mysore, and Tezpur University. The lead author is affiliated with Vaikhari AI, which hosts the benchmark at core.vaikhari.ai."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The lead author is from Vaikhari AI which hosts the benchmark, creating a potential conflict of interest that is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. The lead author's affiliation with Vaikhari AI (which hosts the benchmark) is a potential financial interest that is not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states a model selection cutoff date of January 22, 2026, but does not state training data cutoff dates for the individual models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses contamination mitigation: 'To mitigate evaluation contamination and overfitting risks, different portions serve different purposes.' The blind subset (101 questions) is withheld to reduce contamination risk."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is newly created, and the paper takes steps to mitigate contamination by splitting into open/blind subsets and separating training-purpose data from evaluation data (Section 3.1)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human baseline study with 1,000+ participants has no mention of pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics approval is mentioned for the human baseline study involving 1,000+ participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Participants are described only as 'over 1,000 participants in India, spanning undergraduate to postdoctoral education levels.' No detailed demographics (age, gender, field of study, etc.) are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for the human baseline participants beyond education level range."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not a between-subjects experimental study; all participants took the same benchmark. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states participants 'completed the benchmark under blind evaluation conditions.'"
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on how many participants started vs. finished or whether any were excluded from analysis."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The Ethical Considerations section acknowledges 'significant computational cost and associated carbon emissions' but does not quantify inference costs, API costs, or per-model evaluation time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget, GPU hours, API spend, or hardware details are stated despite evaluating 29 models on 203+ questions."
       }
     }
   },
   "claims": [
     {
       "claim": "29 state-of-the-art LLMs achieve 48.25–70.9% overall accuracy on the CORE benchmark, with near-ceiling performance on related pairs (86.5–100%) but severe degradation on unrelated pairs (0–41.35%).",
       "evidence": "Table 3 and Section 5.1 report these accuracy ranges across 29 models.",
       "supported": "strong"
     },
     {
       "claim": "Models maintain similar confidence (~92–94%) on both related and unrelated pairs despite 40–80 percentage point accuracy differences.",
       "evidence": "Table 3 reports confidence ranges of ~93–95% for related and ~91–94% for unrelated pairs. Section 5.1 and Figure 1 illustrate this.",
       "supported": "strong"
     },
     {
       "claim": "Expected Calibration Error increases 2–4x on unrelated pairs compared to related pairs.",
       "evidence": "Table 4 shows ECE of ~8.0–15.0% for related pairs vs ~24.0–51.0% for unrelated pairs. Section 5.2 and Figure 2.",
       "supported": "strong"
     },
     {
       "claim": "The mean semantic collapse rate of 37.6% indicates systematic generation of spurious relations rather than random guessing.",
       "evidence": "Section 5.3 reports this rate and contrasts it with the 75% expected error from random guessing.",
       "supported": "moderate"
     },
     {
       "claim": "On the 225K MCQ CORE dataset, accuracy drops to approximately 2%.",
       "evidence": "Section 5.5 states this but provides no detailed breakdown, model-by-model results, or methodology specifics for this evaluation.",
       "supported": "weak"
     },
     {
       "claim": "The failure reflects a shared limitation in how current models handle relation absence, potentially from architectural inductive biases.",
       "evidence": "Section 6.1–6.2 provides speculative architectural and training-objective explanations without controlled experiments to support them.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CORE reveals a dramatic asymmetry in LLM reasoning: 29 frontier models achieve 86.5–100% accuracy on related concept pairs but only 0–41.35% on unrelated pairs, while maintaining similar confidence (~92–94%) in both cases. This confidence-accuracy inversion is accompanied by ECE increases of 2–4x on unrelated pairs and a 37.6% mean semantic collapse rate, where models fabricate plausible but spurious relational structures. A human baseline of 1,000+ participants achieves 92.6% overall and 95.1% on unrelated pairs, demonstrating the task is not inherently difficult for humans.",
   "red_flags": [
     {
       "flag": "Small benchmark size",
       "detail": "The core benchmark contains only 203 questions (102 open, 101 blind). Performance differences across models on such a small test set may not be statistically meaningful, and no significance tests or confidence intervals are provided."
     },
     {
       "flag": "Vague 225K dataset evaluation",
       "detail": "The claim of ~2% accuracy on the 225K dataset (Section 5.5) is stated without detailed methodology, model-by-model results, or explanation of how this evaluation was conducted. This is a dramatic claim with minimal supporting detail."
     },
     {
       "flag": "Potential conflict of interest",
       "detail": "The lead author is affiliated with Vaikhari AI, which hosts the benchmark at core.vaikhari.ai. This potential commercial interest in the benchmark's prominence is not disclosed or discussed."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "The paper uses 'recommended inference settings' and 'default configurations' without specifying actual temperature, top-p, or other parameters. Different settings could substantially affect results."
     },
     {
       "flag": "Human baseline methodology gaps",
       "detail": "The 1,000+ participant human baseline lacks IRB approval, detailed demographics, recruitment methods, inclusion criteria, and attrition reporting. The participants are all from India, which may limit generalizability of the human baseline."
     }
   ],
   "cited_papers": [
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Examines whether emergent abilities in LLMs are measurement artifacts, directly relevant to interpreting benchmark performance discontinuities."
     },
     {
       "title": "AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions",
       "authors": ["Polina Kirichenko", "Mark Ibrahim", "Kamalika Chaudhuri", "Samuel J. Bell"],
       "year": 2025,
       "arxiv_id": "2506.09038",
       "relevance": "Evaluates LLM ability to abstain from answering, closely related to CORE's evaluation of unrelatedness recognition."
     },
     {
       "title": "SimpleQA: Measuring Short-form Factuality in Large Language Models",
       "authors": ["Jason Wei"],
       "year": 2024,
       "arxiv_id": "2411.04368",
       "relevance": "Evaluates LLM factuality and refusal rates, complementary to CORE's assessment of spurious relation generation."
     },
     {
       "title": "Towards Understanding Sycophancy in Language Models",
       "authors": ["Mrinank Sharma", "Meg Tong", "Tomasz Korbak"],
       "year": 2023,
       "arxiv_id": "2310.13548",
       "relevance": "Studies sycophancy in LLMs, relevant to understanding why models fabricate relational structures rather than expressing uncertainty."
     },
     {
       "title": "Language Models Don't Always Say What They Think: Unfaithful Explanations in Chain-of-Thought Prompting",
       "authors": ["Miles Turpin", "Julian Michael", "Ethan Perez", "Samuel Bowman"],
       "year": 2023,
       "relevance": "Examines unfaithful reasoning in LLMs, relevant to CORE's finding of coherent but incorrect relational explanations."
     },
     {
       "title": "Holistic Evaluation of Language Models",
       "authors": ["Percy Liang", "Rishi Bommasani", "Tony Lee"],
       "year": 2022,
       "arxiv_id": "2211.09110",
       "relevance": "Major LLM evaluation framework; CORE extends evaluation to unrelatedness reasoning as a new dimension."
     },
     {
       "title": "Language Models (Mostly) Know What They Know",
       "authors": ["Saurav Kadavath", "Tom Conerly", "Amanda Askell"],
       "year": 2022,
       "arxiv_id": "2207.05221",
       "relevance": "Studies LLM calibration and self-knowledge, directly relevant to CORE's finding of systematic miscalibration on unrelated pairs."
     },
     {
       "title": "The Reversal Curse: LLMs trained on 'A is B' fail to learn 'B is A'",
       "authors": ["Lukas Berglund", "Meg Tong", "Max Kaufmann"],
       "year": 2024,
       "relevance": "Demonstrates systematic reasoning failures in LLMs related to relational knowledge, complementary to CORE's findings."
     },
     {
       "title": "GSM-Symbolic: Understanding the limitations of mathematical reasoning in large language models",
       "authors": ["Iman Mirzadeh", "Keivan Alizadeh", "Hooman Shahrokhi"],
       "year": 2025,
       "relevance": "Exposes limitations in LLM reasoning through controlled benchmark variations, similar methodology to CORE's approach."
     },
     {
       "title": "Emergent Analogical Reasoning with Large Language Models",
       "authors": ["Taylor Webb", "Keith J. Holyoak", "Hongjing Lu"],
       "year": 2023,
       "relevance": "Evaluates LLM analogical reasoning capability, directly related to CORE's analogy-based evaluation format."
     }
   ]
 }