ai-research-survey

scan.json (30504B)

---

 {
   "paper": {
     "title": "Not All Metrics Are Guilty: Improving NLG Evaluation by Diversifying References",
     "authors": [
       "Tianyi Tang",
       "Hongyuan Lu",
       "Yuchen Eleanor Jiang",
       "Haoyang Huang",
       "Dongdong Zhang",
       "Wayne Xin Zhao",
       "Tom Kocmi",
       "Furu Wei"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2305.15067",
     "doi": "10.48550/arXiv.2305.15067"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Diversifying references using LLMs (GPT-3.5-turbo-instruct) with diverse linguistic prompts significantly improves correlation between automatic NLG evaluation metrics and human judgments across machine translation (WMT22), text summarization (SummEval), and image captioning (PASCAL-50S). The method is compatible with both traditional metrics (BLEU, ROUGE) and LLM-based metrics (GEMBA, ChatGPT-eval), with maximum aggregation outperforming mean aggregation. Mixing ten diverse prompts covering different linguistic aspects yields better results than any single prompt type, and performance continues to improve up to ~20 references before saturating.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository link provided in the abstract: https://github.com/RUCAIBox/Div-Ref. The paper states 'We release all the code and data at https://github.com/RUCAIBox/Div-Ref to facilitate research.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use publicly available benchmarks (WMT22, SummEval, PASCAL-50S) and release their diversified references at the GitHub repository. The paper states code and data are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or detailed environment setup in the paper. The paper only mentions specific model checkpoints used for individual metrics in Appendix A.1 but no environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While code is released, the paper lacks a 'Reproducing Results' section or specific commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Figures 1-4 and Tables 3-5 report point estimates only (e.g., '77.7' system-level accuracy, '19.4' Kendall Tau). No confidence intervals, error bars, or ± notation anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements ('consistent correlation improvements', 'significant effects') based solely on comparing raw numbers without any statistical significance test. No p-values, t-tests, or bootstrap tests reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3-4 report 'Average Gains' columns showing improvements in context (e.g., '+3.0' system-level, '+2.9' segment-level for Zh-En). Results show baseline and improved values side by side (e.g., BLEU from 14.5 to 19.4), providing sufficient context to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of sample sizes or power analysis. The paper uses existing benchmarks without justifying whether the number of evaluation instances is adequate for the claimed improvements."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported anywhere. Results are single-run despite using stochastic generation (nucleus sampling with top_p=0.9). The paper acknowledges in limitations that 'The OpenAI API also is non-deterministic' but never measures this variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Single-Ref baseline is compared against Div-Ref throughout. Ablation analysis (Section 4.3) also compares against three paraphrasing models (PEGASUS, Parrot, QCPG) and LLaMA-2-70b-chat."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Metrics include contemporary SOTA methods: COMET (2020), BLEURT (2020), GEMBA (2023), ChatGPT-eval (2023). Paraphrasing baselines include QCPG (2022) and LLaMA-2 (2023). All are recent and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents comprehensive ablation across five dimensions: (1) diversifying model choice, (2) instruction prompt design, (3) aggregation function, (4) post-filtering effect, (5) number of diversified references. Results shown in Tables 3-5 and Figure 4."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "16 metrics across 5 categories evaluated: character-based (ChrF), word-based (BLEU, ROUGE variants, METEOR, CIDEr, SPICE), embedding-based (BERTScore, MoverScore), trained (BLEURT, Prism, COMET, BARTScore), and LLM-based (GEMBA, ChatGPT-eval). Listed in Table 2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the diversified references or the method's outputs. The 94.6% quality assessment of generated sentences was done by GPT-3.5, not humans. The paper uses existing human annotations from WMT22/SummEval/PASCAL-50S but does not conduct its own human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses standard held-out benchmark test sets: WMT22 Metrics Shared Task, SummEval, and PASCAL-50S. These are established evaluation benchmarks not used for any tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by language pair (Zh-En, En-De, En-Ru) in Figure 1 and Tables 3-4, by summarization aspect (coherence, consistency, fluency, relevance) in Figure 2, and by caption difficulty setting (HC, HI, HM, MM) in Figure 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure case analysis or examples of where Div-Ref fails to improve or hurts performance. The limitations section mentions generic domain limitations but does not analyze specific failure instances."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Mean aggregation hurts performance in some settings (En-De: -2.8 average gains, Table 4). BERTScore slightly drops in HM setting (90.0→90.3 but not meaningful). Multilingual diverse prompts show 'gains are also not obvious' for non-English. LLM diversifying ability 'in non-English is not as good as that in English.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that diversifying references 'can significantly enhance the correlation between automatic evaluation and human evaluation' are supported by Figures 1-3 showing consistent improvements. The claim of compatibility with LLM-based evaluation is supported by GEMBA and ChatGPT-eval improvements."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim that reference diversification improves correlation is supported by controlled ablation (Section 4.3): systematically varying model, prompts, aggregation, filtering, and number of references while holding other factors constant. This single-variable manipulation constitutes adequate causal evidence for the claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Improving NLG Evaluation by Diversifying References' implies general NLG applicability, but only three specific tasks are tested (translation, summarization, image captioning). The limitations acknowledge 'diverse prompts may fail in specialized domains, such as finance and biomedicine' but the main claims and title are not bounded to tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for why diversified references improve correlation. For instance: does maximum aggregation simply inflate scores to match human tendencies? Could the improvement be an artifact of GPT-3.5 generating references that are more stylistically similar to the hypothesis models? These alternatives are not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly measures correlation between automatic metrics and human judgments (Kendall Tau, Spearman, pairwise accuracy) and frames claims accordingly. They do not overclaim beyond 'improving correlation with human evaluation' — the proxy matches the claim granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'gpt-3.5-turbo-instruct' and 'gpt-3.5-turbo' but these are marketing names without version snapshots or API dates. For evaluation models, specific checkpoints are given (e.g., 'BLEURT-20', 'Unbabel/wmt22-comet-da') but the core LLM used for diversification lacks a pinned version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "All ten diversifying prompts are provided verbatim in Section 3.2.2. The paraphrasing prompt is given in Section 3.2.1. The filtering prompt is given in Section 4.3. These are complete and reproducible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 reports temperature=1, top_p=0.9 for diversification. GEMBA uses text-davinci-003 with temperature=0. ChatGPT-eval uses gpt-3.5-turbo with temperature=0. Number of diversified sentences = 10."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a straightforward prompt-response pipeline for generating paraphrases."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 describes each benchmark's structure and evaluation protocol. Section 4.1.3 describes the diversification pipeline (10 instructions, one sentence per instruction, max aggregation). Appendix A.1 specifies exact implementations and checkpoints for each metric."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the conclusion, discussing multiple specific concerns including domain limitations, cost constraints, non-determinism, and model availability."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: (1) diverse prompts may fail in specialized domains like finance and biomedicine, (2) omission of GEMBA ablation due to text-davinci-003 cost, (3) OpenAI API non-determinism affecting reproducibility, (4) risk of OpenAI removing existing models."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly states what was not tested: specialized domains (finance, biomedicine), the optimal cost-effectiveness trade-off between number of references and evaluation time, and that GEMBA ablation was omitted. It also identifies need for 'fine-grained prompts tailored to address the specific challenges posed by professional terminology.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks used (WMT22, SummEval, PASCAL-50S) are all publicly available. The paper releases diversified references and code at https://github.com/RUCAIBox/Div-Ref. The underlying data can be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 describes each benchmark: WMT22 includes competitor model outputs rated by experts via MQM schema for 3 language pairs; SummEval has 200 summaries from 16 models on CNN/DailyMail with human ratings on 4 aspects; PASCAL-50S has 4,000 triple instances with human preferences."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited. The paper uses existing benchmark data with pre-collected human annotations from prior shared tasks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: (1) take benchmark references, (2) apply 10 diverse prompts via GPT-3.5-turbo-instruct, (3) generate one diversified sentence per prompt, (4) evaluate metrics with max aggregation across original + diversified references, (5) compute correlation with human scores. Section 3.2 and 4.1.3 describe this."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgement section states: 'This work was partially supported by Beijing Natural Science Foundation under Grant No. L233008 and 4222027.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Renmin University of China, Microsoft Research Asia, Microsoft, CUHK, AIWaves Inc. The first author notes 'This work was done during internship at MSRA.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (Beijing Natural Science Foundation) is an academic funding agency with no financial stake in the outcome. While some authors are from Microsoft and the paper uses OpenAI models (Microsoft is an OpenAI investor), the disclosed funder itself is independent."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration in the paper. Authors from Microsoft Research Asia use OpenAI API products but no financial interests are declared or disclaimed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff for gpt-3.5-turbo-instruct is not stated. This matters because if the model was trained on WMT22, SummEval, or PASCAL-50S data, the diversified references could be influenced by memorized benchmark content."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-3.5 may have seen the benchmark references or hypotheses during training. WMT data and SummEval/CNN-DailyMail data are widely available online and likely in GPT-3.5's training set."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks used (WMT22, SummEval based on CNN/DailyMail, PASCAL-50S from 2015) were all publicly available before GPT-3.5's training cutoff. No contamination analysis is performed despite this being directly relevant to the validity of LLM-generated references."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper uses existing benchmark human annotations from prior work."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited or studied."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time reported for generating diversified references. The limitations section mentions 'the high cost of text-davinci-003' for GEMBA but never quantifies the cost of their own diversification method."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated. The paper requires calling GPT-3.5 for every reference sentence across three benchmarks but provides no cost figures, GPU hours, or total API spend."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are not reported across multiple random seeds despite using stochastic generation (nucleus sampling with top_p=0.9). The paper acknowledges API non-determinism in limitations but never measures seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It appears results are from a single generation pass, but this is not explicitly confirmed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The ablation analyzes different settings (models, prompts, aggregation, number of references) but does not state total configurations tried or compute spent on this exploration."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents ablation results for all configurations tested (Tables 3-5, Figure 4), showing why the chosen settings (GPT-3.5, diverse prompts, max aggregation, 10 references) were selected. All alternatives are reported transparently."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed in the paper, so multiple comparison correction is not applicable. The paper compares raw metric values only."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not acknowledge the bias of evaluating their own diversification method. They generate references, evaluate them, and report improvements without discussing potential author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Figure 4 shows performance vs. number of references (0-100) but does not frame this in terms of compute cost. No discussion of the compute trade-off between generating more references and the diminishing returns observed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether correlation with human judgment (Kendall Tau, Spearman) is the right measure of metric quality. The assumption that higher correlation = better metric is taken as given without examining construct validity of the meta-evaluation."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The method is a direct prompt-response pipeline for generating paraphrases."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. GPT-3.5-turbo-instruct's training data likely includes text from WMT shared tasks, SummEval (CNN/DailyMail), and PASCAL-50S, all published years before the model's training cutoff. This could bias the diversified references."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLM having potentially memorized the reference sentences affects the quality and diversity of the generated paraphrases. If GPT-3.5 has seen the benchmarks, the 'diversified' references may be less independent than assumed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the diversified references are truly independent of the benchmark data GPT-3.5 may have been trained on."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied. No canary strings, membership inference, overlap analysis, or decontamination performed."
       }
     }
   },
   "claims": [
     {
       "claim": "Diversifying references using LLMs significantly improves correlation between automatic NLG evaluation metrics and human evaluation across translation, summarization, and captioning tasks.",
       "evidence": "Figures 1-3 show consistent improvements across all three benchmarks: system-level pairwise accuracy improves by +3.0 on average for WMT22 translation (Figure 1a), Spearman correlation improves across all four summarization aspects (Figure 2), and accuracy improves for image captioning in most settings (Figure 3).",
       "supported": "strong"
     },
     {
       "claim": "GPT-3.5-turbo-instruct outperforms traditional paraphrasing models (PEGASUS, Parrot, QCPG) and LLaMA-2-70b-chat for reference diversification.",
       "evidence": "Table 3 shows GPT-3.5 achieves +2.9 average gains at segment level for Zh-En vs +2.0 (PEGASUS), +1.5 (Parrot), +1.5 (QCPG), and +1.4 (LLaMA-2). Tables 3-4 show consistent advantage across language pairs.",
       "supported": "moderate"
     },
     {
       "claim": "Maximum aggregation outperforms mean aggregation for combining multiple reference scores.",
       "evidence": "Tables 3-4 show mean aggregation drops performance in most settings (e.g., -1.1 segment-level Zh-En, -2.8 En-De). The paper explains: 'averaging multiple reference scores could introduce noise from low-quality reference scores' (Section 4.3).",
       "supported": "moderate"
     },
     {
       "claim": "The method is compatible with LLM-based evaluation metrics, further improving their correlation with human judgment.",
       "evidence": "GEMBA improves from 85.8 to 86.1 at system level (Figure 1a). ChatGPT-eval improves across all four summarization dimensions: coherence (+8.4), consistency (+4.7), fluency (+8.4), relevance (+6.6) (Figure 2).",
       "supported": "strong"
     },
     {
       "claim": "94.6% of LLM-generated diversified sentences preserve the original semantics.",
       "evidence": "Section 3.2.3 states: 'We employ another excellent GPT 3.5 to judge whether the generated sentence conveys the same meaning of given reference. The results show that 94.6% of the generated sentences are suitable.'",
       "supported": "weak"
     },
     {
       "claim": "Mixing diverse prompts is better than using any single prompt type repeatedly.",
       "evidence": "Table 5 shows mixing prompts yields +2.9 average gain vs individual prompts ranging from +0.4 to +2.4 on Zh-En segment-level Kendall Tau.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All improvements are claimed based on raw number comparisons (e.g., 14.5→19.4 Kendall Tau) without any significance testing. With stochastic generation and potentially noisy metrics, some claimed improvements may not be statistically significant."
     },
     {
       "flag": "LLM evaluating LLM outputs",
       "detail": "The 94.6% quality figure for generated references comes from GPT-3.5 judging GPT-3.5's own outputs. This is circular evaluation — the model may judge its own outputs more favorably. No human verification of reference quality was performed."
     },
     {
       "flag": "No seed sensitivity or variance analysis",
       "detail": "Despite using stochastic generation (nucleus sampling, top_p=0.9) and a non-deterministic API, no variance across runs is reported. The paper acknowledges this in limitations but all reported results appear to be single-run."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "GPT-3.5 likely saw WMT data, CNN/DailyMail, and PASCAL captions during training. If the model memorized benchmark references, the 'diversified' references may be biased by training data rather than truly exploring the semantic space. This is never discussed."
     },
     {
       "flag": "Potential conflict of interest",
       "detail": "Multiple authors are from Microsoft Research Asia and Microsoft. The paper relies heavily on OpenAI's GPT-3.5 (Microsoft is a major OpenAI investor). The method's value proposition depends on GPT-3.5 being effective, which aligns with Microsoft's commercial interests. This conflict is not acknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models are state-of-the-art evaluators of translation quality",
       "authors": ["Tom Kocmi", "Christian Federmann"],
       "year": 2023,
       "arxiv_id": "2302.14520",
       "relevance": "Demonstrates LLM-as-judge for machine translation evaluation (GEMBA metric), directly relevant to LLM evaluation capabilities."
     },
     {
       "title": "Is chatgpt a good nlg evaluator? a preliminary study",
       "authors": ["Jiaan Wang", "Yunlong Liang", "Fandong Meng"],
       "year": 2023,
       "arxiv_id": "2303.04048",
       "relevance": "Evaluates ChatGPT's capability as an NLG evaluator, relevant to understanding LLM evaluation reliability."
     },
     {
       "title": "Can large language models be an alternative to human evaluations?",
       "authors": ["Cheng-Han Chiang", "Hung-yi Lee"],
       "year": 2023,
       "arxiv_id": "2305.01937",
       "relevance": "Investigates whether LLMs can replace human evaluation, directly relevant to AI evaluation methodology."
     },
     {
       "title": "GPTEval: NLG evaluation using GPT-4 with better human alignment",
       "authors": ["Yang Liu", "Dan Iter", "Yichong Xu"],
       "year": 2023,
       "arxiv_id": "2303.16634",
       "relevance": "Uses GPT-4 for NLG evaluation aligned with human judgments, relevant to LLM-based evaluation approaches."
     },
     {
       "title": "Is chatgpt a good translator? yes with gpt-4 as the engine",
       "authors": ["WX Jiao", "WX Wang", "JT Huang"],
       "year": 2023,
       "arxiv_id": "2301.08745",
       "relevance": "Evaluates ChatGPT/GPT-4 translation capability, relevant to LLM capability assessment methodology."
     },
     {
       "title": "Exploring the use of large language models for reference-free text quality evaluation: A preliminary empirical study",
       "authors": ["Yi Chen", "Rui Wang", "Haiyun Jiang"],
       "year": 2023,
       "arxiv_id": "2304.00723",
       "relevance": "Explores LLMs for reference-free evaluation, relevant to understanding NLG evaluation approaches."
     },
     {
       "title": "BERTScore: Evaluating text generation with BERT",
       "authors": ["Tianyi Zhang", "Varsha Kishore", "Felix Wu"],
       "year": 2020,
       "relevance": "Foundational neural evaluation metric used extensively in LLM-generated text evaluation."
     },
     {
       "title": "Benchmarking large language models for news summarization",
       "authors": ["Tianyi Zhang", "Faisal Ladhak", "Esin Durmus"],
       "year": 2023,
       "arxiv_id": "2301.13848",
       "relevance": "LLM benchmarking study for summarization, relevant to evaluation methodology for LLM capabilities."
     },
     {
       "title": "ChatGPT as a factual inconsistency evaluator for abstractive text summarization",
       "authors": ["Zheheng Luo", "Qianqian Xie", "Sophia Ananiadou"],
       "year": 2023,
       "arxiv_id": "2303.15621",
       "relevance": "Uses ChatGPT for factual consistency evaluation, relevant to LLM-as-judge methodology."
     },
     {
       "title": "Crosslingual generalization through multitask finetuning",
       "authors": ["Niklas Muennighoff", "Thomas Wang", "Lintang Sutawika"],
       "year": 2022,
       "arxiv_id": "2211.01786",
       "relevance": "Cross-lingual LLM capabilities study, relevant to understanding multilingual LLM evaluation."
     }
   ]
 }