ai-research-survey

scan.json (24761B)

---

 {
   "paper": {
     "title": "Thinking Longer, Not Always Smarter: Evaluating LLM Capabilities in Hierarchical Legal Reasoning",
     "authors": ["Li Zhang", "Matthias Grabmair", "Morgan Gray", "Kevin Ashley"],
     "year": 2025,
     "venue": "CSLAW '26",
     "arxiv_id": "2510.08710",
     "doi": "10.1145/3788646.3789522"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "A project page URL is provided (https://thinking-longer-thinking-smarter.vercel.app) but no explicit code repository (GitHub, Zenodo) is linked in the paper. A project page alone is not a confirmed code release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link is provided. The scenarios are generated by their scenario generator but neither the generated instances nor the generator code are explicitly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The evaluation pipeline is described at a high level (Section 5.1) but no runnable scripts or commands are given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows 'Accuracy with 95% CI (n=253)' — confidence intervals are reported for main accuracy results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Model comparisons are made by comparing point estimates (e.g., '64.82% to 92.09%') without any formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context: '2.6-fold improvement' (Section 6.2), '45% increase in computational effort' (Section 8), '3.0-fold increase in computational effort' (Section 8), percentage accuracy with task-level baselines."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "253 test instances per task are used (Section 5.2) but no justification for this number is provided and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are shown in Figure 4, implying variance across instances is captured. Token usage is reported as averages (Tables 1-2)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six models are compared including 5 reasoning LLMs and 1 non-reasoning LLM (qwen3-non-thinking), providing a thinking vs. non-thinking baseline comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include gpt-5-2025-08-07, gemini-2.5-pro-preview-06-05, qwen3-235b-a22b-thinking-2507 — all 2025 frontier models."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No multi-component system is proposed. The paper evaluates off-the-shelf LLMs on a benchmark; there are no system components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: accuracy (exact set match) and reasoning token count (Tables 1-2, Figure 4)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Ground truth is computed by a deterministic symbolic solver (Section 5.1). Human evaluation is irrelevant since correctness is formally verifiable."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Test instances are synthetically generated scenarios (253 per task). No model tuning is performed on any subset — models are evaluated zero/one-shot on all generated instances."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by Task 1, 2, and 3 (Table 1, Figure 4). Table 2 further breaks down token usage by correct vs. incorrect responses."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 provides detailed case study analysis of reasoning failures for qwen3-thinking (verbose/repetitive) and gpt-oss-120b (concise but error-prone), with specific excerpts from thinking traces."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The core finding is negative: models collapse on Task 3 (11.46%-33.99% accuracy). The 'thinking longer doesn't mean thinking smarter' finding is itself a negative result. Section 8 discusses the inefficiency pattern."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (100% Task 1, 64.82%-92.09% Task 2, 11.46%-33.99% Task 3, models spend more tokens on incorrect answers) are all supported by Table 1 and Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — that RL-based post-training reasoning improves accuracy — is supported by comparing thinking vs. non-thinking variants of the same model family (qwen3-thinking 78.66% vs qwen3-non-thinking 30.04% on Task 2). This controlled comparison within one model family is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 explicitly states: 'Our evaluation is currently limited to the domain of trade secret law within the U.S. legal system, as represented by the CATO framework.' Specific scope limitations are acknowledged."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 discusses domain shift from math/logic training to legal reasoning as an alternative explanation for poor performance. Section 9 discusses that models may appear to reason effectively by 'retrieving and adapting existing solutions, masking the underlying deficit.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures exact set match accuracy on formally defined legal reasoning tasks. Claims are about 'LLM capabilities in hierarchical legal reasoning' which directly matches the measurement. The formal framework ensures measurements are well-defined."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers with dates/versions are provided: 'gpt-5-2025-08-07', 'qwen3-235b-a22b-thinking-2507', 'gpt-oss-120b', 'gemini-2.5-pro-preview-06-05', 'gemini-2.5-flash-preview-05-20' (Section 5.3)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates for all three tasks are provided in Appendix C (Boxes C.1, C.2, C.3), including step-by-step instructions, definitions, and output format specifications."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 reports temperature (0.3), top_p (0.95), max_tokens (65,536) for each model family, with footnote explaining rationale and noting gpt-5 uses default parameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are prompted directly with structured inputs and produce JSON outputs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1-5.2 documents the evaluation pipeline: scenario generation from CATO framework, 253 instances per task, cases with 4 factors each, hierarchy in Mermaid format. The ground truth creation process is described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 'Limitations and Future Work' provides substantive discussion of multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 discusses specific threats: limited to trade secret law domain, CATO hierarchy may not capture full legal reasoning complexity, RL training on math/logic tasks may not transfer to legal reasoning, and the fixed case complexity (4 factors)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 10: 'Our evaluation is currently limited to the domain of trade secret law within the U.S. legal system.' Also acknowledges hierarchical structures 'may vary across different areas of law.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (generated scenarios, model outputs, thinking traces beyond excerpts) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the scenario generation process: sampling factors from CATO framework, creating case pairs with specific properties (blocking, downplaying opportunities), and generating ground truth via deterministic solver."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data consists of synthetically generated legal reasoning scenarios from the CATO framework."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 and Section 5.1 document the full pipeline: scenario generation → ground truth creation → LLM inference → evaluation. Each stage is described with its purpose."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (University of Pittsburgh, Technical University of Munich) are clearly listed. Authors are academic researchers with no apparent affiliation to the model providers being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Absence of disclosure is not absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models, despite all being proprietary models with undisclosed training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The CATO framework (factors, hierarchy, case representations) has been published since the 1990s and could be in model training data. This is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While scenarios are newly generated, they use the well-known CATO factor hierarchy which models may have seen. The paper does not discuss whether models' familiarity with CATO concepts could inflate performance. Task 1's 100% accuracy across all models is consistent with contamination on the framework structure."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It evaluates LLMs on synthetically generated scenarios."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token usage (thinking tokens) is reported per model per task in Table 1, and broken down by correct/incorrect responses in Table 2. This is a direct measure of inference cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (API costs, total tokens, wall-clock time for all experiments) is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or runs. Results appear to be from single runs per model per instance."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs per instance is not stated. It appears to be a single run per instance given the lack of any variance-across-runs reporting."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Footnote 1 mentions temperature was set 'based on the models' performance on other reasoning benchmarks' but no search budget (configurations tried, compute spent) is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 and footnote 1 explain the rationale for parameter choices: temperature=0.3 chosen 'based on the models' performance on other reasoning benchmarks to maintain sufficient consistency,' avoiding temperature=0 to 'prevent overly deterministic responses.'"
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple model comparisons are made across three tasks with no statistical tests and therefore no multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the benchmark and evaluation framework. They do not discuss potential bias from evaluating third-party models on their own benchmark design, nor do they have independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 and Figure 4 (right panel) explicitly analyze performance vs. token usage. Section 8 and Figure 5 compare computational effort between correct and incorrect responses. This is a central finding of the paper."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper grounds the benchmark in the established CATO framework (Section 3), discusses the relationship between formal tasks and real legal reasoning (Sections 1, 9), and acknowledges in Section 10 that CATO may not capture full legal reasoning complexity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are prompted directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The CATO framework (1990s publications) and its factor hierarchy are well-documented in AI & Law literature. Models trained on this literature could have internalized the reasoning patterns. This is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The prompts provide full task definitions, factor hierarchies, and one-shot examples. Whether this evaluation setup leaks information beyond what would be available in realistic usage is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All 253 scenarios per task are generated from the same CATO factor set and hierarchy. Structural similarities between instances are not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "All models achieve 100% accuracy on Task 1 (identifying surface-level distinctions) but performance collapses on Task 3 (integrated analysis: 11.46%-33.99%)",
       "evidence": "Table 1 and Figure 4 show 100% accuracy for all 6 models on Task 1, degradation on Task 2 (64.82%-92.09%), and collapse on Task 3 (11.46%-33.99%).",
       "supported": "strong"
     },
     {
       "claim": "Models consistently expend more computational resources on incorrect responses than correct ones",
       "evidence": "Table 2 and Figure 5 show gpt-5 uses 4,456 tokens on incorrect Task 2 responses vs 3,081 for correct ones (45% increase). Pattern is consistent across most models/tasks.",
       "supported": "strong"
     },
     {
       "claim": "Reasoning capabilities from RL-based post-training provide a 2.6-fold improvement on hierarchical reasoning",
       "evidence": "Section 6.2: qwen3-thinking achieves 78.66% on Task 2 vs 30.04% for qwen3-non-thinking. On Task 3, thinking model achieves 33.99% while non-thinking achieves 0.00%.",
       "supported": "strong"
     },
     {
       "claim": "More reasoning tokens do not correlate with better performance across models",
       "evidence": "Section 6.3: qwen3-thinking uses 9,596 tokens on Task 2 for 78.66% accuracy, while gpt-5 uses 3,189 tokens for 92.09% accuracy — 3.0x more tokens for worse performance.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper reveals a performance degradation paradox in LLM hierarchical legal reasoning: while all models achieve 100% accuracy on surface-level distinction identification, performance collapses to 11-34% on integrated multi-step analysis. Models consistently spend more tokens on incorrect answers than correct ones, suggesting 'thinking longer' does not mean 'thinking smarter.' RL-based reasoning training helps (2.6x improvement for thinking vs non-thinking models) but is insufficient for complex integrated reasoning.",
   "red_flags": [
     {
       "flag": "Single-run results",
       "detail": "Results appear to be from single runs per model per instance. With temperature=0.3, outputs are not fully deterministic. No variance across runs is reported, making it impossible to assess result stability."
     },
     {
       "flag": "Potential contamination of CATO framework",
       "detail": "The CATO factor hierarchy has been published since the 1990s. All evaluated models could have seen CATO-related content during training. Task 1's universal 100% accuracy is consistent with models having internalized the framework's structure, yet contamination is never discussed."
     },
     {
       "flag": "No statistical tests for model comparisons",
       "detail": "All claims about model differences (e.g., gpt-5 vs qwen3-thinking, thinking vs non-thinking) are based on comparing point estimates. With n=253, confidence intervals overlap for some comparisons but no formal tests are conducted."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational work on chain-of-thought prompting for LLM reasoning, directly relevant to the paper's evaluation of reasoning capabilities."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Extended reasoning approaches to multi-path exploration; relevant to hierarchical reasoning evaluation."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "RL-based post-training for reasoning, the training paradigm whose effectiveness this paper evaluates."
     },
     {
       "title": "Legalbench: A collaboratively built benchmark for measuring legal reasoning in large language models",
       "authors": ["Neel Guha", "Julian Nyarko", "Daniel Ho"],
       "year": 2023,
       "relevance": "Major legal reasoning benchmark for LLMs; this paper positions itself against it by testing integrated reasoning chains vs isolated skills."
     },
     {
       "title": "GPT-5 System Card",
       "authors": ["OpenAI"],
       "year": 2025,
       "relevance": "System card for one of the evaluated models (gpt-5-2025-08-07), a frontier reasoning model."
     },
     {
       "title": "Gemini 2.5: Pushing the frontier with advanced reasoning, multimodality, long context, and next generation agentic capabilities",
       "authors": ["Gheorghe Comanici"],
       "year": 2025,
       "arxiv_id": "2507.06261",
       "relevance": "Technical report for two of the evaluated models (gemini-pro and gemini-flash)."
     },
     {
       "title": "Qwen3 technical report",
       "authors": ["An Yang", "Anfeng Li"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Technical report for two evaluated models (qwen3-thinking and qwen3-non-thinking)."
     },
     {
       "title": "Sleeper agents: Training deceptive llms that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "AI safety work on deceptive LLM behavior, relevant to trustworthy AI evaluation."
     },
     {
       "title": "Augmented language models: a survey",
       "authors": ["Grégoire Mialon"],
       "year": 2023,
       "arxiv_id": "2302.07842",
       "relevance": "Survey of augmented LM approaches including tool use and reasoning; relevant to understanding LLM reasoning capabilities."
     },
     {
       "title": "A survey of frontiers in llm reasoning: Inference scaling, learning to reason, and agentic systems",
       "authors": ["Zixuan Ke", "Fangkai Jiao"],
       "year": 2025,
       "arxiv_id": "2504.09037",
       "relevance": "Comprehensive survey of LLM reasoning approaches including inference scaling, directly relevant to the 'thinking longer' finding."
     }
   ]
 }