ai-research-survey

scan.json (30584B)

---

 {
   "paper": {
     "title": "StructTest: Benchmarking LLMs' Reasoning through Compositional Structured Outputs",
     "authors": [
       "Hailin Chen",
       "Fangkai Jiao",
       "Mathieu Ravaut",
       "Nawshad Farruque",
       "Xuan Phi Nguyen",
       "Chengwei Qin",
       "Manan Dey",
       "Bosheng Ding",
       "Caiming Xiong",
       "Shafiq Joty",
       "Yingbo Zhou"
     ],
     "year": 2024,
     "venue": "arXiv (Preprint, under review)",
     "arxiv_id": "2412.18011",
     "doi": "10.48550/arXiv.2412.18011"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link in the abstract: 'Code & Data: https://github.com/SparkJiao/StructTest'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub link is labeled 'Code & Data', indicating the benchmark data is released. The paper also uses publicly available datasets (e.g., GSM8K) for some subtasks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link exists but the paper itself contains no README-like instructions for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (1-4) report point estimates only with no confidence intervals, error bars, or ± notation. Results are presented as single accuracy percentages."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'DeepSeek-R1 consistently outperform others') based solely on comparing raw accuracy numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Pearson correlations (92.5% with Arena, 96.3% with MMLU) and relative performance differences (e.g., '70% drop in accuracy on Hard tasks compared to Easy ones,' 'error rates as high as 84%'). Full accuracy tables provide baseline context for all comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The number of test examples per task is not stated in the paper, and no justification for sample sizes is provided. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multi-run spread measures are reported. All results appear to be single-run evaluations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 17 LLMs that serve as mutual baselines. Results are also compared to external benchmarks (ChatBot Arena, MMLU) in Table 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include models current as of late 2024: DeepSeek-R1, GPT-4o, Claude-3.5-sonnet, Gemini-1.5-pro. Appendix A confirms inference was performed Nov 27 – Dec 14, 2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Figure 5 shows per-domain correlation analysis, but no formal ablation study tests which benchmark components contribute most to discriminative power. They do not remove domains or difficulty levels to measure their individual contribution to overall benchmark validity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results include per-task accuracy (exact match, pass rates), aggregate Easy/Hard/All scores, and Pearson correlation with external benchmarks (Arena and MMLU). Error rate analysis by features is also provided (Figures 2-4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All assessment is rule-based and programmatic, which the paper positions as a feature. However, validating the rule-based evaluator against human judgments would strengthen the benchmark."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Models are evaluated as-is on StructTest without any tuning. For math tasks, standard GSM8K test splits are used. Section 3.5 mentions plans for a confidential held-out test set for future iterations."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns in Tables 1-3: by domain (Summarization, Code, HTML, Math), difficulty (Easy/Hard), and individual subtask (e.g., 7 summarization formats, 4 code tasks). Figure 5 shows per-domain correlation."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 analyzes GPT-4o error rates by bullet point count and length. Figure 3 shows tag-count error distributions for HTML. Figure 4 shows format-specific error rates for math. Section 4.1 discusses specific failure patterns (e.g., indentation collapse, long output formatting failures)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "While the paper reports that models fail on the benchmark (which is expected by design), it does not report negative results about its own methodology — e.g., task designs that didn't work, evaluation rules that needed revision, or approaches tried and abandoned."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that StructTest is challenging (supported by Table 1: best model at 74.76%), provides unbiased evaluation (rule-based by design), and serves as a proxy for reasoning (supported by 92.5% correlation with Arena in Table 4) are supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims models 'may have overfitted to specific formats and styles' (Section 4.1) based on performance drops when format changes, but doesn't rule out alternative explanations (e.g., novel task difficulty independent of overfitting). Ablation studies or controlled experiments would be needed to support this causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to benchmark 'LLMs' Reasoning' but the benchmark measures structured output generation and instruction following across 4 domains. The abstract claims it is 'a robust proxy for measuring reasoning capabilities' — this is a broad generalization from format-following to general reasoning that is not adequately bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its key findings. For the high correlation with Arena/MMLU, it doesn't consider that both may simply measure instruction-following ability rather than reasoning. For format overfitting claims, it doesn't consider that novel/complex tasks are inherently harder regardless of memorization."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures structured output compliance and calls it 'reasoning.' While Section 1 argues structured outputs require 'decomposing instructions, understanding and retaining subtle constraints during extended decoding, and executing logical actions,' there is no rigorous analysis distinguishing format-following ability from reasoning ability. The word 'proxy' is used but the gap is not adequately discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 (Appendix A) provides exact API versions for all closed-source models (e.g., 'gpt-4o-2024-08-06', 'claude-3-5-sonnet-20241022', 'gemini-1.5-pro-002'). Open-source models are identified by specific model names and sizes (e.g., 'Llama-3.1-70B', 'DeepSeek-R1')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt examples are provided for every task type in Appendix B (Figures 6-17), including actual prompt text with format instructions, examples, and response templates."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No generation hyperparameters are reported — temperature, top-p, max tokens, and other sampling settings are not mentioned anywhere in the paper."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are prompted directly with task instructions and generate responses in a single pass."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.4 describe task construction in detail: use of Python ast package for code tasks, uniform sampling of parameters from fixed intervals for HTML/summarization, 7 final answer styles and 5 CoT styles for math, and the construction of Easy/Hard splits."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section. Section 4.3 discusses updating the benchmark and Section 3.5 discusses contamination robustness, but neither constitutes a substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address issues like whether format-following correlates with reasoning only for current model families, whether the 14-model correlation analysis is statistically underpowered, or whether the benchmark's difficulty comes from genuine reasoning demands vs. unfamiliar formatting."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper claims StructTest is a proxy for 'general reasoning ability' and 'comprehensive model evaluation' without stating what it does not measure — e.g., creative generation, multi-turn dialogue, factual knowledge, or long-context reasoning."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark tasks are released via GitHub, but raw model outputs (predictions for each of the 17 LLMs on each task) are not explicitly stated to be available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.4 describe task construction procedures in detail. The sources of underlying data are specified (e.g., GSM8K for math, code snippets for code tasks). Evaluation rules are formalized with mathematical notation (Equations 1-4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (GSM8K) and programmatically constructed tasks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While task construction rules are described, the paper does not report exact counts of test examples per task, filtering criteria applied to source data, or the full pipeline from data collection to final benchmark composition."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Nanyang Technological University, Salesforce Research, I2R A*STAR Singapore, and University of Alberta."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Without funding disclosure, independence cannot be assessed. Multiple authors are from Salesforce Research, which has commercial interests in AI/LLM technology, though Salesforce models are not directly evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the 17 evaluated models. Only the inference date range is provided (Nov 27 – Dec 14, 2024)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 extensively discusses contamination resistance: 'its tasks are designed to be highly unlikely to have been encountered by existing models during training.' The paper also notes that 'reliance on underlying data from existing benchmarks in some subtasks...raises concerns about data memorization' (Section 1)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 addresses contamination directly, arguing that StructTest's compositional design and data-decoupled evaluation mitigate contamination risk. The paper also acknowledges that underlying datasets like GSM8K may be memorized, and argues that format requirements make memorization insufficient for high scores."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency figures are reported despite evaluating 17 models. The paper claims StructTest is 'cost-effective' and 'cheap to run' but never quantifies the actual cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states inference was performed Nov 27 – Dec 14, 2024 (Appendix A) but does not quantify total compute: no GPU hours, API spend, or hardware specifications."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run evaluations. No multi-seed analysis or seed sensitivity reporting is present."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per model is never stated. It appears each model was evaluated once."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Generation settings (temperature, sampling) are not even reported, let alone the search budget for finding them."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation is provided for how model generation settings were chosen. The paper does not discuss whether default or tuned settings were used."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 17 models across multiple domains and difficulty levels without any statistical tests or correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the benchmark and evaluated all models on it. They do not discuss potential bias in benchmark design that could favor certain model architectures or training approaches."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 1.5B parameters (DeepSeek-R1-Distill-Qwen-1.5B) to frontier closed-source models are compared without any discussion of compute differences or performance per compute unit."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims StructTest measures 'reasoning' but actually measures structured output generation and instruction following. The connection between format compliance and reasoning is asserted ('skills closely aligned with complex reasoning') rather than empirically validated. The Pearson correlation with Arena/MMLU is presented as evidence but could reflect shared instruction-following ability rather than reasoning."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are prompted directly in a single-turn evaluation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 and Section 1 discuss temporal leakage: 'The reliance on underlying data from existing benchmarks in some subtasks, coupled with notable performance drops, raises concerns about data memorization rather than genuine generalization.' The paper argues StructTest's format requirements mitigate memorization effects."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context or provides hints not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not verify whether code snippets or other source data overlap with model training data. While Section 3.5 argues contamination is unlikely by design, no empirical independence verification is performed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection methods (canary strings, membership inference, n-gram overlap analysis) are applied. The contamination argument is purely by design."
       }
     }
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "claims": [
     {
       "claim": "StructTest remains challenging even for top-performing models, with DeepSeek-R1 achieving only 74.76% overall and 67.10% on Hard tasks.",
       "evidence": "Table 1 shows DeepSeek-R1 at 74.76% StructTest-All and 67.10% StructTest-Hard. GPT-4o scores 73.16% overall, 57.29% Hard. Many models collapse below 10% on specific Hard subtasks.",
       "supported": "strong"
     },
     {
       "claim": "StructTest achieves Pearson correlation of 92.5% with ChatBot Arena and 96.3% with MMLU, validating it as a proxy for general reasoning.",
       "evidence": "Table 4 and Section 4.2 report correlations computed over 14 models with both Arena scores and MMLU accuracy.",
       "supported": "moderate"
     },
     {
       "claim": "Existing math reasoning comparisons between models are likely unreliable and unfair unless tested across diverse formats.",
       "evidence": "Figure 4 shows GPT-4o error rates vary from 0% to 84% across 20 Hard math formats on GSM8K. Section 4.1 notes models scoring 90%+ on standard GSM8K drop to 60-77% on StructTest math.",
       "supported": "moderate"
     },
     {
       "claim": "Open-source DeepSeek-R1 is comparable to top closed-source models on structured output tasks.",
       "evidence": "Table 1 shows DeepSeek-R1 (74.76%) competitive with GPT-4o (73.16%) and Claude-3.5-sonnet (72.62%), outperforming them on Hard splits (67.10% vs 57.29% and 53.69%).",
       "supported": "strong"
     },
     {
       "claim": "StructTest is robust to data contamination due to its compositional design decoupled from underlying data.",
       "evidence": "Section 3.5 argues by design: tasks are compositional and unlikely to appear in training data, benchmark can be regularly updated. No empirical contamination testing is performed.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "StructTest introduces a rule-based, programmatically verifiable benchmark evaluating 17 LLMs across 4 structured output domains (summarization, code, HTML, math). Even top models like DeepSeek-R1 achieve only 74.76% overall, with many models collapsing on Hard tasks. The benchmark correlates strongly with ChatBot Arena (r=0.925) and MMLU (r=0.963) over 14 models. Performance on format-constrained math tasks drops dramatically compared to standard benchmarks (e.g., Gemini-1.5-pro from 91.7% standard to 77.3% Easy / 73.4% Hard), suggesting models may have overfitted to specific answer formats.",
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All 17 models are evaluated with single-run results across all tasks. No confidence intervals, standard deviations, or multi-run variance are reported anywhere in the paper."
     },
     {
       "flag": "Correlation claim on underpowered sample",
       "detail": "The headline claim of 92.5% Pearson correlation with ChatBot Arena is computed on only 14 data points. With n=14, this correlation has very wide confidence intervals and is sensitive to individual outliers, but no uncertainty estimate is provided."
     },
     {
       "flag": "Construct validity gap: format-following framed as reasoning",
       "detail": "The paper measures instruction-following and structured output compliance but frames this as 'reasoning.' The high correlation with Arena (which also measures instruction-following/helpfulness) could reflect shared instruction-following ability rather than reasoning. No experiment isolates the reasoning component from format compliance."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. It does not discuss what StructTest cannot measure, whether the benchmark's difficulty reflects genuine reasoning demands vs. unfamiliar formatting, or the statistical limitations of the 14-model correlation analysis."
     },
     {
       "flag": "Missing generation hyperparameters",
       "detail": "Temperature, top-p, max tokens, and other generation settings are not reported for any of the 17 models. These settings significantly affect output quality and could confound comparisons between models."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLMs on real-world software engineering tasks, directly relevant to LLM code generation evaluation."
     },
     {
       "title": "Chatbot Arena: An open platform for evaluating llms by human preference",
       "authors": ["Wei-Lin Chiang", "Lianmin Zheng", "Ying Sheng"],
       "year": 2024,
       "arxiv_id": "2403.04132",
       "relevance": "Human-based LLM evaluation platform used as the primary external validation benchmark for StructTest's correlation analysis."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt"],
       "year": 2020,
       "relevance": "MMLU benchmark, the second external validation target for StructTest and a widely used LLM reasoning evaluation."
     },
     {
       "title": "How much are LLMs contaminated? A comprehensive survey and the LLMSanitize library",
       "authors": ["Mathieu Ravaut", "Bosheng Ding", "Fangkai Jiao", "Hailin Chen"],
       "year": 2024,
       "relevance": "Survey on LLM benchmark contamination, directly motivating StructTest's contamination-resistant design."
     },
     {
       "title": "FOFO: A benchmark to evaluate LLMs' format-following capability",
       "authors": ["Congying Xia", "Chen Xing", "Jiangshu Du"],
       "year": 2024,
       "relevance": "Prior work on evaluating LLMs' format-following ability, the most directly related benchmark to StructTest."
     },
     {
       "title": "Cheating automatic LLM benchmarks: Null models achieve high win rates",
       "authors": ["Xiaosen Zheng", "Tianyu Pang", "Chao Du"],
       "year": 2024,
       "relevance": "Demonstrates vulnerability of model-based evaluation benchmarks, motivating StructTest's rule-based approach."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "MT-Bench and the LLM-as-judge paradigm, representing model-based evaluation that StructTest aims to complement."
     },
     {
       "title": "Length-controlled AlpacaEval: A simple way to debias automatic evaluators",
       "authors": ["Yann Dubois", "Balázs Galambosi", "Percy Liang", "Tatsunori B. Hashimoto"],
       "year": 2024,
       "arxiv_id": "2404.04475",
       "relevance": "Addresses length bias in model-based evaluation, a key limitation that StructTest's rule-based approach avoids."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "relevance": "Contamination-free code benchmark sharing StructTest's goal of contamination-resistant LLM evaluation."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian"],
       "year": 2021,
       "relevance": "GSM8K benchmark used as the underlying data source for StructTest's math reasoning evaluation tasks."
     },
     {
       "title": "LLMs are biased towards output formats! Systematically evaluating and mitigating output format bias of LLMs",
       "authors": ["Xuan Long Do", "Hai Nguyen Ngoc", "Tiviatis Sim"],
       "year": 2024,
       "relevance": "Studies how format instructions influence LLM task performance, closely related to StructTest's core investigation."
     },
     {
       "title": "Struc-bench: Are large language models really good at generating complex structured data?",
       "authors": ["Xiangru Tang", "Yiming Zong", "Jason Phang"],
       "year": 2023,
       "relevance": "Prior benchmark for evaluating LLMs on structured generation, including HTML, directly preceding StructTest's approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The benchmark and evaluation code are released and could be used by practitioners to evaluate LLMs, though it requires setup and is primarily a research tool."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that models lose up to 70% accuracy with format constraints and may be overfitting to popular answer formats is mildly surprising but aligns with growing intuition about benchmark fragility."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised by this benchmark evaluation paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicit 'current benchmarks are unreliable' angle with format overfitting claims, but no strong controversy or direct challenges to specific parties."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data released on GitHub (github.com/SparkJiao/StructTest), enabling users to evaluate their own models."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From Salesforce Research and NTU; evaluates well-known models (GPT-4o, DeepSeek-R1, Claude-3.5-sonnet) but the benchmark itself is not yet widely known."
     }
   }
 }