ai-research-survey

scan.json (32774B)

---

 {
   "paper": {
     "title": "ReLE: A Scalable System and Structured Benchmark for Diagnosing Capability Anisotropy in Chinese LLMs",
     "authors": [
       "Rui Fang",
       "Jian Li",
       "Wei Chen",
       "Bin Hu",
       "Ying-Cong Chen",
       "Xin Tang",
       "Liang Diao"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.17399",
     "doi": "10.48550/arXiv.2601.17399"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 7.1 states 'We plan to release an anonymized subset of the failure case repository, along with the evaluation scripts and configuration files.' This is a promise of future release, not an actual release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark data (207,843 samples) is not released. Section 7.1 mentions planned release of scripts and a filtered failure case repository. The Domain-Specific Private Set (20%) is proprietary. No download links are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specifications are provided. The system architecture is described conceptually (Section 3) but without sufficient detail to recreate the evaluation environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the system architecture and methodology but does not include commands, scripts, or a README for reproducing the evaluation."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports ±SD for all domain scores. Section 6.1 reports bootstrap 95% confidence intervals for RSA: [10.2, 12.6] for ReLE vs [4.1, 5.9] for baselines, from 1,000 bootstrap samples."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 reports Kolmogorov-Smirnov test (D = 0.42, p < 10^-5) for RSA distribution differences. Bootstrap resampling with 1,000 iterations is used for confidence intervals. p-values reported (p < 0.001, p < 0.01)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported throughout: 70% cost reduction ($69K → $20.7K), RSA 11.4 vs ~5.0, ρ = 0.96 ranking correlation, Cohen's κ = 0.81 judge-human agreement, Anisotropy Index 0.74, Healthcare gap ~12 points."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 304 models across 207,843 samples but provides no power analysis or explicit justification for these sample sizes. The 50-model control experiment subset (Section 6.1.1) is also not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports ±SD for all domain scores across model categories. The Neyman allocation strategy explicitly estimates and uses stratum variance (S²_h,m). Bootstrap confidence intervals provide variance estimates for RSA."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares ReLE against GLUE, HELM, and OpenCompass. Section 6.1 compares RSA values against C-Eval and CLUE baselines under identical reweighting protocols."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include OpenCompass (2024), LiveBench (2024), Arena-Hard (2024), Chatbot Arena (2024), and CompassJudger (2025). These are recent and representative of the state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 provides a control experiment comparing dynamic sampling vs. full-set evaluation, but this validates one component. There is no systematic ablation of the system's core components (hybrid scoring stages, prompt schema adapter, variance-aware scheduler) to isolate their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Rank Stability Amplitude (RSA), Capability Inconsistency (CI), Anisotropy Index, Spearman correlation (ρ), Cohen's κ, Pearson r, end-to-end latency, and cost metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 reports Cohen's κ = 0.81 between GPT-4o judge and human annotators on 500 adversarial samples. Human expert review validates 10% of perturbed dataset samples (Section 4.1). Inter-annotator agreement ≥96.8% for prompt schema."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes a Private Anchor Set (PAS) of 5,000 newly constructed samples 'strictly isolated from the internet.' The Domain-Specific Private Set (20%) serves as 'a strictly held-out test set.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-domain breakdown across 7 core domains and 3 model categories. Table 2 shows the Domain × Capability matrix. Figure 2 shows per-dimension radar plots. Section 6.3 breaks down failure patterns by domain."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 presents systematic failure pattern analysis from 2.1M+ failure cases, identifying three patterns: Popular Benchmark Overfitting (41%), Domain-Dependent Gaps, and Size-Independent Gaps (18% of ≥20B models). Figure 4 visualizes failure distribution."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report approaches that were tried and abandoned, or configurations of their own system that failed. All design choices are presented as successful. The negative findings about evaluated models (overfitting, size-independent gaps) are about the subjects, not the method."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 70% cost reduction (supported by Section 3.3/6.4), ρ = 0.96 ranking correlation (Table 4), RSA 11.4 vs ~5.0 (Section 6.1), and 207,843 samples across 304 models (Section 4) are all substantiated in the paper body."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper mostly uses appropriate language: correlational ('correlates more with instruction tuning, r = 0.65, than scale, r = 0.48'). The system-level causal claim (dynamic scheduling reduces cost) is validated with the control experiment in Table 4. Section 6.1.1 explicitly isolates sampling noise from anisotropy."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope to Chinese LLMs and 7 domains. Section 7.2 states 'emerging areas like Industrial IoT are not yet included.' Section 7.1 notes the architecture is 'language-agnostic' but is 'instantiated with Chinese-language benchmarks in this work.' The paper positions itself as diagnostic, not comprehensive."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1.1 explicitly tests whether RSA is an artifact of sampling noise vs genuine anisotropy (94.8% structural, 5.2% noise). Section 5.1 explains the 'Agent Score Anomaly' as format alignment rather than capability. Section 6.1 considers whether instability is a 'statistical artifact.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central argument IS that aggregate scores are misleading proxies. Section 1.1 explicitly argues single-score rankings create 'an illusion of progress.' The RSA metric quantifies the gap between aggregate ranking (proxy) and structured capability assessment (outcome). Section 5.1 distinguishes 'latent capability and interface compliance.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The judge model is specified as 'GPT-4o-0513' (specific version), and secondary judges as 'Qwen-Max' and 'Claude-3.7-Sonnet.' However, the 304 evaluated models are referenced by marketing names (e.g., 'Llama 3', 'Gemini-3-Pro', 'DeepSeek-V2/V3') without specific version dates or API snapshots."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 describes the Unified Prompt Schema conceptually (fields for Input Content, Output Format Requirements, Domain Tags, model-specific adapters) but does not provide actual prompt text. No appendix with prompts. Only natural-language descriptions of what prompts contain."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Some system parameters are reported: scoring thresholds (0.92, 0.60), smoothing constant ε = 0.1, pilot set size 5%. However, key LLM inference parameters (temperature, top-p, max tokens) are not stated for either the evaluated models or the judge model."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "ReLE is a benchmark evaluation system, not an agentic scaffold. While it evaluates agent tasks, the system itself does not use agentic scaffolding (no tool use, retry logic, or memory management)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents three data sources with percentages (45% fresh, 35% adapted, 20% private). Section 4.3 describes the multi-level decontamination pipeline: 13-gram overlap filtering, semantic deduplication via BGE-M3 (threshold > 0.85), symbolic solver validation (SymPy/WolframAlpha), and 10% human expert review."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 'Limitations & Future Directions' is a dedicated subsection discussing specific limitations including domain coverage, mechanism understanding, and multi-modal evaluation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 identifies specific limitations: only 7 domains covered (IoT missing), multi-modal evaluation is limited, internal mechanisms causing anisotropy require further study. Section 6.1.1 tests whether RSA instability is a measurement artifact. Section 3.2 addresses judge bias specifically."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it covers Chinese LLMs across 7 domains (Section 7.2), positions ReLE as 'not a replacement for comprehensive static benchmarks' (Abstract/Section 1), and notes the framework is 'instantiated with Chinese-language benchmarks in this work' (Section 7.1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Section 8 mentions plans to 'open-source ReLE's failure case repository (2.1M instances) and evaluation infrastructure' but this is a future intention. No download links or supplementary data are currently provided. The private domain set (20%) will never be available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes three data sources: Dynamic Fresh Set (June 2024–Jan 2026), Solver-Verified Academic Refinement from Math24O etc., and Domain-Specific Private Set from industry partners. Section 4.3 describes real exam questions (2025 Gaokao), 56 domain experts for custom samples, and adapted academic datasets."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "56 domain experts are mentioned for creating custom domain samples (Section 4.3) but their recruitment, qualifications, and selection process are not described. Industry partners providing the private dataset (20%) are unnamed ('finance/medical')."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: data sourcing (3 sources with percentages) → value perturbation for academic sets → symbolic solver validation → semantic deduplication (BGE-M3) → 13-gram overlap filtering → human expert review (10%) → Private Anchor Set isolation. Section 4.3 provides filtering criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section, acknowledgments, or grant information is provided. Authors are affiliated with commercial entities (Huawei, NSFOCUS Technologies, Ping An Insurance) but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Sun Yat-sen University, HKUST-GZ, NSFOCUS Technologies Co. Ltd, Huawei, and Ping An Property & Casualty Insurance Company of China. All affiliations appear on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. Authors from Huawei and other companies may have products among the 304 evaluated models, creating a potential undisclosed conflict. Without funding disclosure, independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Authors from Huawei and NSFOCUS (commercial AI/security companies) may have financial interests related to model evaluation rankings. Absence of disclosure is not absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 mentions 'zero overlap with training data cutoffs of models released before mid-2025' for the fresh set, acknowledging cutoffs exist. However, specific training data cutoff dates for the 304 evaluated models are not stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 extensively discusses train/test overlap: 13-gram overlap checks against CommonCrawl/C4, embedding-based semantic deduplication (cosine > 0.85 threshold against pre-October 2025 web snapshots), and Private Anchor Set (PAS) with Generalization Gap analysis (∆g > 15% flags overfitting)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 addresses contamination with a 45% Dynamic Fresh Set (June 2024–Jan 2026) ensuring zero overlap with pre-mid-2025 training data. Section 4.3 implements multi-level decontamination (n-gram + semantic). The Private Held-out Validation set provides contamination-resistant validation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates AI models, not human participants. The 56 domain experts are content creators for the benchmark, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects research is conducted. The study evaluates LLM capabilities on benchmark tasks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Model characteristics (commercial vs. open-source, parameter scale) are reported instead."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Model inclusion criteria (304 models covering diverse architectures and scales) are described in Section 4.4."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions assigned to humans."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The evaluation is automated."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants to experience attrition."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Costs are a central focus: $69,000 for traditional full-set evaluation vs. $20,700 for ReLE (70% reduction). Average cost of $230 per model is stated (footnote 1, Section 1). Price-performance analysis (1-5 yuan models) in Section 5.2. Latency analysis in Section 6.4."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total evaluation cost is stated: $20,700 for 304 models with dynamic sampling vs. $69,000 for full-set evaluation. Section 6.4 discusses compute efficiency as a first-class evaluation dimension."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seed sensitivity analysis. Model evaluations appear to be single-pass. The bootstrap resampling (1,000 iterations) is for statistical testing of RSA, not for testing seed sensitivity of model evaluations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs per model is not explicitly stated. Bootstrap iterations (1,000) are stated for statistical tests but these are resampling analyses, not separate model evaluation runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "System parameters (scoring thresholds 0.92/0.60, ε = 0.1, pilot set 5%) appear manually set. No hyperparameter search budget, search method, or total compute spent on system parameter tuning is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of system configuration (scoring thresholds, sampling parameters, weighting schemes) is not justified through systematic comparison of alternatives. The three weighting schemes (General, Professional, Reasoning) are presented without explaining why these specific schemes were chosen."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 304 models across 22 dimensions and 7 domains but does not mention multiple comparison correction (Bonferroni, Holm, etc.). While some p-values are very small (p < 10^-5), the paper makes many comparisons without addressing family-wise error rate."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors present their own benchmark and evaluate models on it without acknowledging the potential bias of designing and scoring a benchmark to demonstrate specific findings (e.g., that RSA is higher in ReLE than baselines). No independent evaluation or bias mitigation discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 analyzes price-performance tradeoffs: '1-5 yuan models achieve comparable performance to high-priced models (≥5 yuan) in 8 out of 22 dimensions.' Section 6.4 reports latency analysis and non-linear cost-performance relationships. Section 3.3 reports cost savings explicitly."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Construct validity is a central theme. The paper argues aggregate scores mask capability anisotropy (Section 1.1), introduces the Anisotropy Index to measure dimensional independence (Section 5.1), distinguishes 'latent capability and interface compliance' (Section 5.1), and uses the Generalization Gap to flag benchmark overfitting (Section 4.3)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 describes a Unified API-First Evaluation Protocol that standardizes the access layer across all 304 models. Section 3.1 describes a Model-Specific Adapter Layer ensuring 'performance differences reflect reasoning capability rather than instruction-following failures.' Section 5.1 explicitly discusses format alignment confounds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 addresses temporal leakage with a Dynamic Fresh Set (45%) of samples authored June 2024–Jan 2026, 'ensuring zero overlap with training data cutoffs of models released before mid-2025.' Temporal segmentation is a core design principle."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation setup leaks answer information through context, prompt formatting, or task structure. Feature leakage (whether input features contain information unavailable at prediction time) is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses adapted academic datasets (35%) with value perturbation and symbolic solver validation, but does not explicitly discuss whether the adapted samples maintain independence from training data beyond surface-level perturbation. Structural similarities between original and perturbed samples are not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 applies concrete detection methods: 13-gram overlap checks against CommonCrawl/C4, embedding-based semantic deduplication (BGE-M3, cosine > 0.85 against pre-October 2025 web snapshots), and Private Anchor Set (PAS) with Generalization Gap computation (∆g > 15% flags overfitting)."
       }
     }
   },
   "claims": [
     {
       "claim": "ReLE's dynamic variance-aware scheduler reduces evaluation costs by 70% compared to full-set evaluation while maintaining ranking correlation of ρ = 0.96.",
       "evidence": "Table 4 shows dynamic sampling yields RSA 11.4 vs full-set 10.8, with Spearman ρ = 0.96 (p < 10^-5). Cost: $20,700 vs $69,000 (Section 1.2, 3.3).",
       "supported": "strong"
     },
     {
       "claim": "Models exhibit significantly higher ranking instability in ReLE (mean RSA 11.4) versus traditional benchmarks (RSA ~5.0), demonstrating capability anisotropy.",
       "evidence": "Section 6.1: 65% of models show RSA ≥10, 23% show RSA ≥20. KS test D = 0.42, p < 10^-5. Bootstrap 95% CI: ReLE [10.2, 12.6] vs baselines [4.1, 5.9]. Control experiment (Table 4) shows 94.8% of instability is structural.",
       "supported": "strong"
     },
     {
       "claim": "LLM capability is highly anisotropic (Anisotropy Index = 0.74), meaning performance in one domain is a poor predictor of performance in another.",
       "evidence": "Section 5.1 computes Ianiso from average Pearson correlation across 22 dimensions. Cross-dimensional correlations between professional and general domains are low (r = 0.26). Visualized in Figure 2 radar plots.",
       "supported": "moderate"
     },
     {
       "claim": "41% of evaluated models exhibit signs of overfitting to public benchmarks, scoring high on C-Eval (avg 73.2) but failing on ReLE's professional sub-tasks (avg 48.5).",
       "evidence": "Section 6.3. However, the gap could reflect task difficulty differences rather than overfitting. No formal overfitting test (e.g., comparing generalization gap distributions) is provided beyond the aggregate score difference.",
       "supported": "weak"
     },
     {
       "claim": "Agent-specialized models significantly outperform general-purpose commercial models in Tool Use (74.8 vs 62.4), driven by instruction tuning rather than parameter scale.",
       "evidence": "Table 3 shows the score gap. Section 5.2 reports correlation with instruction tuning (r = 0.65) vs scale (r = 0.48). Section 5.1 attributes the gap to 'Format Alignment' with function-calling schemas.",
       "supported": "moderate"
     },
     {
       "claim": "Mid-priced commercial models (1-5 yuan) match expensive models (≥5 yuan) in 8 of 22 dimensions with average score difference ≤3.2%.",
       "evidence": "Section 5.2 states this finding. However, detailed price-performance data (which 8 dimensions, specific model comparisons) is not shown in tables.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ReLE evaluates 304 Chinese LLMs across 207,843 samples in a Domain × Capability matrix, revealing significant capability anisotropy (Ianiso = 0.74) masked by aggregate scores. Rankings are highly unstable under different capability weightings (mean RSA 11.4 vs ~5.0 in traditional benchmarks, p < 10^-5), with 94.8% of instability attributable to structural capability differences rather than measurement noise. The system's variance-aware adaptive scheduler reduces evaluation costs by 70% while preserving ranking fidelity (ρ = 0.96). Notably, 41% of models show benchmark overfitting patterns and agent-specialized models outperform general models in tool use through instruction tuning rather than parameter scale.",
   "red_flags": [
     {
       "flag": "Undisclosed funding and potential conflicts",
       "detail": "Authors are affiliated with Huawei, NSFOCUS Technologies, and Ping An Insurance — companies that may have models among the 304 evaluated. No funding disclosure, competing interests statement, or conflict-of-interest acknowledgment is provided."
     },
     {
       "flag": "Overfitting claim conflates task difficulty with contamination",
       "detail": "Section 6.3 claims 41% of models show 'Popular Benchmark Overfitting' based on scoring higher on C-Eval (avg 73.2) than ReLE professional sub-tasks (avg 48.5). This gap could simply reflect that niche professional tasks are inherently harder than general benchmarks, not that models are overfitted. No formal overfitting test is provided."
     },
     {
       "flag": "No code, data, or artifact release",
       "detail": "Despite evaluating 304 models across 207,843 samples with a claimed 2.1M+ failure case repository, nothing is released. All claims rely on trust in the authors' system. The 20% proprietary dataset from unnamed industry partners can never be verified."
     },
     {
       "flag": "Judge model evaluating itself",
       "detail": "GPT-4o-0513 is used as the primary judge for semi-objective tasks (24% of scoring). GPT-4o variants are among the 304 evaluated models. While concordance with secondary judges (Pearson r = 0.88) is reported, the primary judge may still have systematic preferences for its own family's output patterns."
     },
     {
       "flag": "Selective reporting of price-performance data",
       "detail": "The claim that '1-5 yuan models match expensive models in 8 of 22 dimensions' (Section 5.2) does not show which 8 dimensions, which specific models, or the full price-performance analysis. This makes the claim difficult to verify."
     }
   ],
   "cited_papers": [
     {
       "title": "Holistic evaluation of language models",
       "authors": ["P. Liang", "R. Bommasani", "T. Lee"],
       "year": 2022,
       "relevance": "Foundational LLM evaluation framework (HELM); directly compared as baseline for ReLE's diagnostic approach."
     },
     {
       "title": "C-Eval: A comprehensive evaluation of Chinese language models",
       "authors": ["Y. Huang", "C. Liang", "X. Du"],
       "year": 2021,
       "relevance": "Major Chinese LLM benchmark used as baseline for RSA comparison, demonstrating benchmark saturation."
     },
     {
       "title": "OpenCompass: A Universal Evaluation Platform for Foundation Models",
       "authors": ["OpenCompass Contributors"],
       "year": 2024,
       "arxiv_id": "2307.06233",
       "relevance": "Leading open-source Chinese LLM evaluation platform; directly compared against ReLE's design philosophy."
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "authors": ["W.-L. Chiang", "L. Zheng", "Y. Sheng"],
       "year": 2024,
       "arxiv_id": "2403.04132",
       "relevance": "Preference-based LLM evaluation framework; contrasted with ReLE's structured diagnostic approach."
     },
     {
       "title": "LiveBench: A Challenging, Contamination-Free LLM Benchmark",
       "authors": ["C. White", "S. Dooley", "S. Manjunatha"],
       "year": 2024,
       "arxiv_id": "2406.19314",
       "relevance": "Dynamic benchmark addressing contamination via continuous updates; compared with ReLE's adaptive approach."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "relevance": "Cost-aware LLM usage framework; aligns with ReLE's efficiency-first evaluation philosophy."
     },
     {
       "title": "CompassJudger-1: All-in-one Judge Model Helps Model Evaluation and Evolution",
       "authors": ["OpenCompass Team"],
       "year": 2025,
       "arxiv_id": "2410.16256",
       "relevance": "Judge-model-based evaluation method; discussed as orthogonal to ReLE's structured evaluation approach."
     },
     {
       "title": "From Live Data to High-Quality Benchmarks: The Arena-Hard Pipeline",
       "authors": ["T. Li", "W.-L. Chiang", "E. Frick"],
       "year": 2024,
       "arxiv_id": "2406.11939",
       "relevance": "Dynamic benchmark pipeline from live data; compared with ReLE's adaptive evaluation approach."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["AI @ Meta"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Major open-source LLM family; cited as example of models that have saturated traditional benchmarks."
     },
     {
       "title": "Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators",
       "authors": ["Y. Dubois", "X. Li", "R. Taori"],
       "year": 2024,
       "arxiv_id": "2404.04475",
       "relevance": "Addresses evaluator bias in LLM assessment, relevant to ReLE's judge bias mitigation approach."
     },
     {
       "title": "CMMLU: Measuring massive multitask language understanding in Chinese",
       "authors": ["H. Li", "Y. Zhang", "F. Koto"],
       "year": 2023,
       "arxiv_id": "2306.09212",
       "relevance": "Chinese-specific multitask LLM benchmark; part of the evaluation landscape ReLE addresses."
     },
     {
       "title": "AlignBench: Benchmarking Chinese Alignment",
       "authors": ["Y. Liu", "Y. Duan", "Y. Zhang"],
       "year": 2024,
       "arxiv_id": "2311.18743",
       "relevance": "Chinese LLM alignment benchmark; compared with ReLE's multi-dimensional decomposition approach."
     }
   ]
 }