ai-research-survey

scan.json (27597B)

---

 {
   "paper": {
     "title": "The Geometry of Thought: How Scale Restructures Reasoning in Large Language Models",
     "authors": ["Samuel Cyrenius Anderson"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.13358",
     "doi": "10.48550/arXiv.2601.13358"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Scale induces domain-specific geometric reorganizations in LLM reasoning representations rather than uniform improvement. Legal reasoning 'crystallizes' at 70B (45% dimensionality collapse, 31% alignment increase), scientific/math reasoning remains geometrically invariant, and code reasoning forms discrete strategic clusters. A universal oscillatory signature (coherence ≈ −0.4) persists across all domains and scales, suggesting an architectural invariant of transformers.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Paper states code will be released 'upon acceptance' and is 'available to qualified researchers upon request' — not currently publicly available."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used (GSM8K, GPQA, HumanEval, CaseHOLD, LexGLUE-SCOTUS) are publicly available under their respective licenses, as stated in the Data and Code Availability section."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (8× NVIDIA B200 GPUs, bfloat16) and HuggingFace Transformers is mentioned, but no requirements.txt, library versions, or environment specification is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the methodology but does not include scripts, commands, or a README-style guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 9 reports 95% bootstrapped confidence intervals for ∆Alignment across domains (e.g., Law: +0.22 [0.14, 0.30])."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal significance tests (p-values, t-tests, etc.) are reported. Confidence intervals are shown for alignment changes but comparative claims between domains lack formal statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported throughout with baseline context: 45% dimensional collapse (d95: 501→274), 31% alignment increase (0.72→0.94), 213% clustering increase (silhouette: 0.133→0.417), and percentage changes with invariance zones."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes are stated (GSM8K N=7473, CaseHOLD N=5000, GPQA N=500, HumanEval N=164) but no justification is given for why these sizes are adequate, especially the small N=164 for code and N=500 for science."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy decoding makes generation deterministic, but operator training results report only point accuracy (63.6%) with no variance across seeds or runs. Alignment variance (σ=0.027) is reported for trajectory statistics but not for the operator learning experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Operator learning experiments include baselines: identity operator (ĥT = h0) and mean predictor (ĥT = E[hT]). Three operator architectures are compared (Linear, MLP, DeepONet). Section 3.5.1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are simple mathematical baselines (identity, mean predictor). No comparison to existing inference acceleration methods (speculative decoding, early exit) or prior trajectory analysis work is made experimentally."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple operator architectures are compared (Linear, MLP, DeepONet, Spectral KAN, Turbo), and the Turbo operator's velocity-conditioning (h1−h0) is evaluated as a component. Section 2.8 discusses why Spectral KAN underperforms."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six geometric metrics are reported: global dimension (d95), intrinsic dimension (dmle), trajectory alignment, step-to-step coherence, silhouette score, and G/L ratio. Operator evaluation uses MSE, latent similarity, and probe decoding accuracy."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a geometric analysis of hidden-state trajectories; human evaluation of system outputs is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Operator learning uses a fixed 70/15/15 train/val/test split with seed 42, and accuracy is reported on held-out test trajectories. Section 3.5.1."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by domain (Law, Science, Code, Math) and by scale (8B, 70B), with per-domain metrics across all six geometric dimensions. Figure 4 provides the full metrics matrix."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Liquid phase (Science, Math) is presented as a null result where scale does not reorganize geometry. LogicBench extraction failure is acknowledged. The Spectral KAN operator's underperformance is discussed. Section 4.2 discusses why science resists crystallization."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: Spectral KAN underperforms simpler Turbo operator, scientific/mathematical reasoning shows no geometric improvement with scale, LogicBench extraction did not complete. Section 2.8 explicitly frames KAN as an informative negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (45% dimensionality collapse, 31% alignment increase, 10× manifold untangling, silhouette 0.13→0.42, coherence ≈−0.4, 63.6% accuracy) are all supported with corresponding figures and numerical results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses causal language throughout: 'scale triggers Crystallization', 'scale reshapes reasoning', 'scale induces domain-specific phase transitions.' These are observational comparisons across two model sizes, not controlled causal interventions. The paper acknowledges 'correlation, not causation' in Section 4.7 but continues using causal framing in the conclusion."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 (Limitations) explicitly bounds generalization: single model family (Llama-3), English-only data, two scale points, dataset confounds. The paper acknowledges these prevent claims about universal transformer behavior."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 presents two competing interpretations (Expertise vs. Compression) for crystallization. Section 4.7 discusses dataset confounds as an alternative explanation. Section 4.2 proposes domain structure (open vs. closed systems) as explanation for the liquid state."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to distinguish geometric measurements (dimensionality, alignment, coherence) from reasoning quality. It does not claim geometry equals capability; it claims geometry predicts operator learnability, which is separately tested. The probe decoding accuracy is presented as a proxy with clear framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model identifiers given: 'meta-llama/Meta-Llama-3-8B-Instruct (hidden size d=4096, 32 layers)' and 'meta-llama/Llama-3.1-70B-Instruct (hidden size d=8192, 80 layers)'. Section 3.2.1."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described in natural language ('Prompts instruct step-by-step reasoning and require an explicit delimiter Final:') but the actual prompt text is not provided in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Decoding: greedy (do_sample=False), max_new_tokens=512. Operator training: AdamW lr=1e-4, cosine annealing, batch size 64, 50 epochs, seed 42. MLE: k=10 nearest neighbors. Section 3.3.1 and 3.5.1."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper extracts hidden states from standard model inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the two-pass generate-then-extract protocol, trajectory indexing, delimiter localization, filtering criteria (discard empty generations), minimum-length requirements, and float16→float32 casting. Section 3.2.2 states subsampling sizes."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 is titled 'Limitations' and provides substantive discussion across five specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 lists specific threats: single model family (Llama-3), English-only data, only two scale points preventing characterization of transition dynamics, dataset confounds (size, difficulty, format differ across domains), correlation vs. causation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 explicitly states what results do NOT show: cannot claim generality beyond Llama-3, cannot distinguish sharp transition from smooth compression with only two scale points, cannot claim causal relationships. English-only limitation noted for legal reasoning specifically."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper states it will not 'bundle large raw hidden-state dumps' and code/data will only be available upon acceptance or request. Raw trajectory data is not publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the full trajectory extraction procedure: deterministic greedy decoding, teacher-forced hidden-state capture, trajectory indexing relative to prompt boundary, memory-mapped storage in float16."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data sources are standard public benchmarks (GSM8K, GPQA, HumanEval, CaseHOLD, LexGLUE-SCOTUS)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from dataset loading through tokenization, generation, teacher-forced extraction, trajectory indexing, filtering, and geometric analysis is documented across Sections 3.3-3.4. Filtering criteria and minimum-length requirements are stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. The author is affiliated with Scrivly.AI but no funding source is mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is listed as 'Scrivly.AI' with contact email sam@scrivly.ai."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of funder cannot be assessed. The author's company Scrivly.AI could potentially benefit from inference acceleration findings."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. The author is affiliated with Scrivly.AI which may have commercial interests related to the inference acceleration applications discussed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for Llama-3-8B-Instruct or Llama-3.1-70B-Instruct, despite evaluating them on benchmarks that existed before training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models may have seen GSM8K, HumanEval, GPQA, or LexGLUE data during training, despite these being public benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and GSM8K (2021) were published well before Llama-3's training. The paper does not discuss contamination risk, though it claims to use benchmarks for geometric characterization rather than leaderboard reporting."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or wall-clock time is reported for trajectory extraction or operator training, despite using 8× B200 GPUs and processing 25,000+ trajectories."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (8× NVIDIA B200 GPUs, 180GB VRAM each) but total compute budget (GPU hours, training time, total extraction time) is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Generation uses greedy decoding (deterministic), but operator training uses a single seed (42) with no sensitivity analysis across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of operator training runs is not stated. It appears to be a single run per architecture. Best checkpoint selected by validation MSE but no multi-run statistics."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The operator architectures use fixed hyperparameters (lr=1e-4, batch 64, 50 epochs) with no indication of how these were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Best checkpoint selected by validation MSE on the held-out validation split (15% of data), and reported on test split. Section 3.5.1."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all operator baselines and architectures themselves. No acknowledgment of potential bias from implementing and tuning their own baselines."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The operator architectures differ in parameter count but compute differences are small (all are lightweight adapters). The main comparison is about geometric structure, not compute-performance tradeoffs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses using benchmarks for geometric characterization rather than leaderboard-style evaluation: 'We use these splits as a fixed corpus for geometric characterization and operator/probe training rather than leaderboard-style benchmark reporting.' Section 3.2.2."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The paper extracts hidden states from standard model inference without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage despite using benchmarks (HumanEval 2021, GSM8K 2021) published well before Llama-3's training. Though the paper uses benchmarks for geometric analysis not accuracy evaluation, contamination could still affect hidden-state structure."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, the legal classification task provides the full court opinion and asks for issue area classification — no discussion of whether prompt format affects trajectory geometry."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap or structural similarity between the benchmark datasets and model training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Legal reasoning undergoes 'Crystallization' at scale: 45% collapse in global dimensionality (d95: 501→274), 31% alignment increase (0.72→0.94), and 10× manifold untangling (G/L: 9.82→0.98) between 8B and 70B parameters.",
       "evidence": "Figures 2, 3, 5, 7 and Section 4.1 present the dimensional, alignment, and G/L metrics. 95% CI for alignment gain: [0.14, 0.30] (Figure 9).",
       "supported": "moderate"
     },
     {
       "claim": "Scientific and mathematical reasoning are geometrically invariant to scale (Liquid phase): d95 237→235 for Science, 501→501 for Math.",
       "evidence": "Figures 2, 3, 10. Math alignment change CI spans zero: −0.01 [−0.07, +0.05] (Figure 9).",
       "supported": "moderate"
     },
     {
       "claim": "Code reasoning forms a discrete Lattice of strategic modes with silhouette score increasing from 0.13 to 0.42 and optimal clusters expanding from k=2 to k=5 at scale.",
       "evidence": "Figures 1, 4. Silhouette scores and cluster counts reported in Results section.",
       "supported": "moderate"
     },
     {
       "claim": "A universal oscillatory signature (coherence ≈ −0.4) is invariant across all domains and scales, suggesting an architectural invariant of transformer dynamics.",
       "evidence": "Figure 8 shows coherence values: Law −0.40/−0.40, Science −0.40/−0.40, Code −0.42/−0.40, Math −0.42/−0.40.",
       "supported": "moderate"
     },
     {
       "claim": "Neural Reasoning Operators achieve 63.6% accuracy on held-out legal classification tasks via probe decoding, exceeding baselines by 10 percentage points.",
       "evidence": "Section 1.4 and 5 (Conclusion) report 63.6% accuracy and 74.7% latent similarity for the Turbo operator. Trained on 8B trajectories with 70/15/15 split.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Single model family",
       "detail": "All experiments use Llama-3-Instruct only. The three-phase taxonomy (Crystal/Liquid/Lattice) may reflect Llama-specific training dynamics rather than universal properties of transformer reasoning. The paper acknowledges this but frames findings broadly."
     },
     {
       "flag": "Only two scale points",
       "detail": "With only 8B and 70B comparisons, the paper cannot distinguish a sharp phase transition from gradual compression, yet uses phase transition framing extensively. Intermediate scales (14B, 33B) would be needed to characterize the transition shape."
     },
     {
       "flag": "Operator validation limited to pilot on 8B",
       "detail": "The headline 63.6% operator accuracy is achieved only on 8B legal trajectories with probe decoding. No 70B operator results are reported despite Crystallization at 70B being the central finding. The paper speculates 70B would be better but provides no evidence."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Uses public benchmarks (HumanEval 2021, GSM8K 2021) that likely appeared in Llama-3 training data. While the paper claims to study geometry rather than accuracy, if models have memorized solutions, the hidden-state trajectories may reflect retrieval rather than reasoning, undermining the entire geometric analysis."
     },
     {
       "flag": "Dataset confounds across domains",
       "detail": "Domains differ dramatically in dataset size (164 to 7473), answer format, and difficulty distribution. The paper acknowledges this but does not control for it — observed geometric differences may reflect data characteristics rather than domain properties."
     },
     {
       "flag": "Causal language from observational design",
       "detail": "Despite acknowledging 'correlation, not causation' in limitations, the paper consistently uses causal framing: 'scale triggers', 'scale induces', 'scale reshapes.' Title and abstract use causal framing throughout."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["J. Kaplan", "S. McCandlish", "T. Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper establishing power-law relationships between model parameters, data, and performance."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["J. Hoffmann", "S. Borgeaud", "A. Mensch"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling analysis showing optimal compute allocation requires scaling data and parameters in tandem."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Introduces chain-of-thought prompting showing intermediate reasoning steps improve LLM performance on reasoning tasks."
     },
     {
       "title": "Emergent abilities of large language models",
       "authors": ["J. Wei", "Y. Tay", "R. Bommasani"],
       "year": 2022,
       "arxiv_id": "2206.07682",
       "relevance": "Documents capabilities that appear discontinuously at specific scale thresholds, relevant to phase transition analysis."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["R. Schaeffer", "B. Miranda", "S. Koyejo"],
       "year": 2023,
       "arxiv_id": "2304.15004",
       "relevance": "Challenges emergence framing, arguing apparent discontinuities may reflect metric choice — relevant context for phase transition claims."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark used in this study for code generation evaluation."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["S. Yao", "D. Yu", "J. Zhao"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Structures reasoning as search over alternative continuations, relevant to understanding reasoning trajectory geometry."
     },
     {
       "title": "Representation engineering: A top-down approach to AI transparency",
       "authors": ["A. Zou", "L. Phan", "S. Chen"],
       "year": 2023,
       "arxiv_id": "2310.01405",
       "relevance": "Demonstrates linear probes can identify high-level concepts in activation space, foundational to representation analysis approaches."
     },
     {
       "title": "GPQA: A graduate-level google-proof Q&A benchmark",
       "authors": ["D. Rein", "B. L. Hou", "A. C. Stickland"],
       "year": 2023,
       "arxiv_id": "2311.12022",
       "relevance": "Expert-level scientific reasoning benchmark used in this study."
     },
     {
       "title": "Fast inference from transformers via speculative decoding",
       "authors": ["Y. Leviathan", "M. Kalman", "Y. Matias"],
       "year": 2023,
       "arxiv_id": "2211.17192",
       "relevance": "Key inference acceleration method that this paper positions against with its endpoint prediction approach."
     }
   ]
 }