ai-research-survey

scan.json (24320B)

---

 {
   "paper": {
     "title": "Metric assessment protocol in the context of answer fluctuation on MCQ tasks",
     "authors": ["Ekaterina Goliakova", "Xavier Renard", "Marie-Jeanne Lesot", "Thibault Laugel", "Christophe Marsala", "Marcin Detyniecki"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.15581",
     "doi": "10.48550/arXiv.2507.15581"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Most existing MCQ evaluation metrics strongly correlate with full fluctuation rates (R² > 0.8), even when computed only on the original option order. Probability mass is the best single-permutation proxy for fluctuation. A novel metric, worst accuracy, achieves the highest combined correlation with both full fluctuation rates and original accuracy when computed on cyclic permutations (R² = 0.942). Continuous metrics like probability mass and Brier score show no improvement from additional permutations, while discrete metrics like sensitivity gap and partial fluctuation rates are unstable on small permutation subsets.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available Meta evaluation datasets (HuggingFace meta-llama/Llama-3.1-8B-evals) and publicly available models. The benchmarks (ARC-C, CSQA, MMLU, AGIEval, Winogrande) are all public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using HuggingFace transformers with bfloat16 precision but provides no requirements.txt, library versions, or detailed environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, scripts, or README are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 1 report only R² point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims certain metrics correlate more strongly than others but uses only R² comparisons without significance tests for the differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "R² values are reported throughout (Tables 1a-1c), providing magnitude of correlation effects. Specific values like R² = 0.942 for worst accuracy on cyclic permutations give clear effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 10 models and 17 benchmarks were chosen, and no power analysis for whether this sample is sufficient for the correlation analyses."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 and Figure 4 report standard deviations of metrics across 100 random permutation pairs/tuples, providing spread measures for metric stability analysis."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares its novel worst accuracy metric against multiple existing metrics: average accuracy, PriDe, strong accuracy, sensitivity gap, probability mass, Brier score, entropy, and partial fluctuation rates."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include PriDe (Zheng et al., 2024), strong accuracy (Gupta et al., 2024), and fluctuation rates (Wei et al., 2024), all from 2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies the permutation subset type (original, original+inverse, cyclic, random-2, random-|L|) to assess how each metric performs under different computation budgets, functioning as an ablation over the permutation dimension."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper compares 9 metrics. Evaluation uses R² for correlation with fluctuation rates, correlation with original accuracy, and combined correlation."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant — this paper evaluates statistical properties of metrics, not human-facing outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning system being trained/tested; it evaluates metrics on benchmark results. No train/test split is applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by permutation type (Tables 1a-1c), by benchmark (Figures 3-15 in appendix), and by model (Figures 5-6). Standard deviations are shown per benchmark in Figure 4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses Winogrande as an outlier case where metrics like sensitivity gap and worst accuracy perform poorly due to having only 2 options (Section 5.1). The instability of sensitivity gap and partial fluctuation rates on Rrandom2 is also discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that adding permutations does NOT improve continuous metrics (probability mass, Brier score), that sensitivity gap performs very poorly on Rrandom2 (R² = 0.235), and that partial fluctuation rates lose information about original accuracy."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that most metrics correlate with fluctuation rates and that worst accuracy shows highest association are supported by Tables 1a-1c and Figures 3a-3b."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes correlational claims (metrics correlate with fluctuation rates) and does not claim causal relationships."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Limitations) explicitly states bounds: only option order permutations tested, only models below 10B parameters, only next-token prediction (not text generation). The title specifies 'MCQ tasks'."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that Winogrande's outlier behavior may be due to having only 2 options (Section 5.1), that continuous metrics' stability may explain their capped correlation (Section 5.3), and considers the impact of permutation choice on metric stability."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit that R² correlation with full fluctuation rates is a proxy for actual metric quality, and that a good metric should capture both fluctuation and original accuracy. The protocol (Section 4) clearly distinguishes between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 lists specific models: Llama-3.1-8B, Gemma-2-9B, Mistral-7B-v0.3, Qwen2.5-7B, R1-Distill-Llama-8B, R1-Distill-Qwen-7B, with both pre-trained and instruct-tuned versions specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly shared Meta evaluation datasets that contain 'full final prompts, including instructions, few-shot examples, their order, and option typography' (Section 4.2), with a link to the HuggingFace dataset. The prompt format is shown in Section 4.2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions bfloat16 precision and random seed 0 for permutation sampling, but does not report temperature, top-p, or other inference hyperparameters. Since they use next-token probabilities (greedy/argmax), this is partially implicit but not explicitly stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates next-token probabilities directly."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes using Meta's pre-formatted evaluation datasets. Appendix B provides benchmark details (number of questions, number of options). The handling of AGIEval nan options is documented (footnote 8)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations & Future Work' provides substantive discussion of four specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: only option order permutations (not paraphrases, typography, label changes), only models below 10B, only next-token prediction (not text generation), and unresolved optimal permutation selection."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what was NOT tested: other permutation types, larger models, text generation approaches. The paper also bounds scope in Section 4.1 ('models with parameter sizes below 10B')."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw model outputs (per-question probabilities and answers across all permutations) are not released. Only aggregated R² results are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the full protocol: models are run on all permutations of each benchmark question, next-token probabilities are extracted, and metrics are computed from these. The benchmarks and their sources are specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The 6-step protocol (Section 4) documents the pipeline from benchmark selection through permutation generation, model inference, metric computation, and correlation analysis."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper, despite authors having both academic (Sorbonne/CNRS) and corporate (AXA) affiliations."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Sorbonne University/CNRS/LIP6, AXA, and Polish Academy of Science."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "AXA (an insurance company) co-affiliates several authors. While AXA has no direct stake in MCQ metric evaluation, the potential funding relationship is undisclosed, making independence unverifiable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained LLMs on public benchmarks (MMLU, ARC-C, etc.) but does not state training data cutoff dates for any of the 10 models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models may have seen the benchmark questions during training, despite using well-known public benchmarks like MMLU and ARC-C."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU (2021), ARC-C (2018), CSQA (2019), Winogrande (2021), AGIEval (2024) are all public benchmarks that could be in training data of 2024-2025 models. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses computation cost conceptually (full permutations are 'extremely costly') but never reports actual GPU hours, wall-clock time, or cost per experiment."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated despite running 10 models × 17 benchmarks × all permutations (up to 120 per question for 5-option benchmarks)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 analyzes sensitivity to random permutation selection using 100 random pairs and 100 random tuples, reporting standard deviations across these samples (Figure 4)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states full permutations are computed (|L|! per question). For random analysis, '100 random pairs' and '100 random tuples of size |L|' are explicitly stated (Section 5.3)."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter tuning is performed; the paper evaluates metrics on model outputs without training."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection; all metrics are compared on the same data."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple metrics are compared across multiple permutation types without correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose worst accuracy and compare it favorably against existing metrics without acknowledging potential author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper framework compares metrics at different computational costs (original only vs. 2 permutations vs. |L| permutations vs. all permutations), effectively showing performance as a function of compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 extensively discusses what MCQ benchmarks actually measure vs. what they claim, including sensitivity to permutation, typography, and format changes. The paper's core contribution is questioning whether standard metrics capture what they claim (robustness)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; models are queried directly for next-token probabilities."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Models from 2024-2025 are tested on benchmarks from 2018-2024. No discussion of whether these models may have been trained on benchmark solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setup leaks information. The few-shot examples provided in prompts could potentially bias toward certain answer patterns."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data and test benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Most existing MCQ metrics correlate strongly with full fluctuation rates, even when computed on the original option order only.",
       "evidence": "Table 1a and Figure 3a show R² values of 0.833-0.893 for most metrics on Roriginal, with probability mass achieving R² = 0.893.",
       "supported": "strong"
     },
     {
       "claim": "Worst accuracy achieves the highest combined correlation with both full fluctuation rates and original accuracy when using cyclic permutations.",
       "evidence": "Table 1c shows worst accuracy achieves R² = 0.942 on Rcyclic for the combined target, the highest value in that row.",
       "supported": "strong"
     },
     {
       "claim": "Continuous metrics (probability mass, Brier score) do not improve with additional permutations.",
       "evidence": "Comparing Figures 3a and 3b, probability mass R² remains at 0.894 across Roriginal and Rcyclic. Table 1a confirms this stability.",
       "supported": "strong"
     },
     {
       "claim": "Sensitivity gap and partial fluctuation rates are unstable when computed on only two permutations.",
       "evidence": "Table 1a shows sensitivity gap drops from R² = 0.640 (Roi) to 0.235 (Rrandom2), and partial FR drops from 0.829 to 0.479. Figure 4 shows high standard deviations for these metrics.",
       "supported": "strong"
     },
     {
       "claim": "PriDe achieves the highest correlation with original accuracy on Roi and Rcyclic.",
       "evidence": "Table 1b shows PriDe at R² = 0.993 (Roi) and 0.994 (Rcyclic), slightly exceeding average accuracy.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination discussion",
       "detail": "The paper evaluates 10 LLMs on well-known public benchmarks (MMLU, ARC-C, CSQA) without any discussion of benchmark contamination. Since the paper studies answer fluctuation patterns, contamination could systematically affect which questions show fluctuation (memorized answers may be more stable), biasing the metric comparisons."
     },
     {
       "flag": "Limited model size range",
       "detail": "All 10 models are 7-9B parameters. The authors acknowledge this limitation but the findings may not generalize to larger models that are more commonly used in practice and may exhibit different fluctuation patterns."
     },
     {
       "flag": "No code or raw data released",
       "detail": "Despite the paper proposing a new metric and evaluation protocol, no code or raw experimental data is released, preventing independent verification of the R² calculations."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models sensitivity to the order of options in multiple-choice questions",
       "authors": ["Pouya Pezeshkpour", "Estevam Hruschka"],
       "year": 2024,
       "relevance": "Core prior work on MCQ option order sensitivity and sensitivity gap metric, directly evaluated in this paper."
     },
     {
       "title": "Large Language Models Are Not Robust Multiple Choice Selectors",
       "authors": ["Chujie Zheng", "Hao Zhou", "Fandong Meng", "Jie Zhou", "Minlie Huang"],
       "year": 2024,
       "relevance": "Introduces PriDe debiasing technique and demonstrates MCQ selection bias in LLMs."
     },
     {
       "title": "Unveiling selection biases: Exploring order and token sensitivity in large language models",
       "authors": ["Sheng-Lun Wei", "Cheng-Kuang Wu", "Hen-Hsen Huang", "Hsin-Hsi Chen"],
       "year": 2024,
       "relevance": "Introduces fluctuation rates metric and studies label/option order sensitivity in LLMs."
     },
     {
       "title": "Changing Answer Order Can Decrease MMLU Accuracy",
       "authors": ["Vipul Gupta", "David Pantoja", "Candace Ross", "Adina Williams", "Megan Ung"],
       "year": 2024,
       "relevance": "Introduces strong accuracy metric and studies answer order effects on MMLU."
     },
     {
       "title": "Quantifying Variance in Evaluation Benchmarks",
       "authors": ["Lovish Madaan", "Aaditya K. Singh", "Rylan Schaeffer"],
       "year": 2024,
       "relevance": "Studies variance in LLM evaluation benchmarks and probability mass as a metric."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Demonstrates that continuous metrics reveal different patterns than accuracy for tracking model capabilities."
     },
     {
       "title": "Do LLMs Exhibit Human-like Response Biases? A Case Study in Survey Design",
       "authors": ["Lindia Tjuatja", "Valerie Chen", "Tongshuang Wu", "Ameet Talwalkwar", "Graham Neubig"],
       "year": 2024,
       "relevance": "Studies LLM response biases on MCQ tasks compared to human biases, introduces entropy metric."
     },
     {
       "title": "When Benchmarks are Targets: Revealing the Sensitivity of Large Language Model Leaderboards",
       "authors": ["Norah A. Alzahrani", "Hisham A. Alyahya"],
       "year": 2024,
       "relevance": "Studies benchmark sensitivity including option typography effects on LLM evaluation."
     },
     {
       "title": "Holistic Evaluation of Language Models",
       "authors": ["Percy Liang", "Rishi Bommasani", "Tony Lee"],
       "year": 2023,
       "relevance": "Demonstrates few-shot example variation effects on LLM evaluation scores."
     }
   ]
 }