ai-research-survey

scan.json (25205B)

---

 {
   "paper": {
     "title": "Beyond Static Datasets: A Deep Interaction Approach to LLM Evaluation",
     "authors": ["Jiatong Li", "Rui Li", "Qi Liu"],
     "year": 2023,
     "venue": "Under Review (arXiv preprint)",
     "arxiv_id": "2309.04369"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our source code is available at https://anonymous.4open.science/r/DeepEval-112F' in the abstract. This is an anonymized repository link, which provides a working URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: MBPP for code generation and IWSLT 2017 for machine translation. The idiom database and public goods game are custom but the code repository likely contains them. The public datasets used are standard and referenced."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency list is provided in the paper. There is no mention of library versions or setup instructions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the code repository is linked, the paper does not include a 'Reproducing Results' section or describe how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The public goods game results are presented as boxplots in Figure 4 showing the distribution across 10 runs, including quartiles and ranges. However, the other tasks (Code Review, Machine Translation, Idiom Solitaire) report only point estimates in tables without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'GPT-4 reaches the state-of-the-art performance') but no statistical significance tests are applied. Differences between models are compared by eyeballing scores in tables without any p-values or formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper presents raw scores but does not contextualize the magnitude of differences between models using any effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes. The public goods game uses 10 repeated runs, idiom solitaire uses 30 randomly sampled idioms, and code/translation use portions of MBPP and IWSLT 2017, but no rationale is provided for these choices."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The public goods game boxplots in Figure 4 show variance across 10 runs including quartiles, median, min, and max. However, for Code Review, Machine Translation, and Idiom Solitaire, only mean scores are reported without any variance or standard deviation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare DeepEval against any existing evaluation method (e.g., MT-Bench, AlpacaEval, standard benchmarks). It only compares LLMs against each other within its own framework, without validating whether its rankings are consistent with established evaluation approaches."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included, so there are no contemporary baselines to assess. The paper mentions MT-Bench, PandaLM, AlpacaEval, and GameEval in related work but does not compare against any of them."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The framework has multiple components (message pool, referee, synchronous algorithm, symmetric vs. asymmetric tasks) but none are individually ablated to demonstrate their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses different metrics across tasks: payoff in public goods game, winning rate and successful hit in idiom solitaire, programmer score and reviewer score in code review, and translator score and proofreader score in machine translation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The referee/judge in DeepEval is an LLM, and the paper does not validate the LLM-judge's ratings against human judgments. This is a significant gap since the paper proposes an alternative to human-based evaluation but never validates against it."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe a held-out test set or train/test split. MBPP and IWSLT data are used for evaluation but no mention is made of how splits were handled or whether any tuning was performed on a separate dev set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per task (public goods game, idiom solitaire, code review, machine translation), per role (programmer vs. reviewer, translator vs. proofreader), and per language pair (DE-EN, EN-FR, EN-ZH) in Tables 3-6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 (Case Study) and Appendix A.2 show specific failure cases, including Claude 2 failing at idiom solitaire (producing a non-idiom response) and translation errors. Figure 5 illustrates specific failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every experiment shows the framework producing differentiated rankings. The paper does not discuss any configurations, task designs, or settings that failed or produced unreliable results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the framework can evaluate LLMs through deep interaction in real-world tasks and demonstrates effectiveness through experiments on four tasks. The results sections cover all four tasks with quantitative results. The abstract is relatively modest in its claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims such as 'GPT-4 and Claude 2 have a better decision-making ability' and 'ChatGPT is stronger than GPT-4' in Chinese tasks. These are causal attributions about model abilities based on performance in specific game settings without controlling for confounds like prompt sensitivity or game-specific strategies."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to evaluate 'LLMs in real-world scenarios' and to be 'a general evaluation method that can be applied to a host of real-world tasks,' but tests on only four specific tasks (a game theory game, a Chinese word game, code review on MBPP, and machine translation on IWSLT). No explicit bounds are stated on what settings these results do or do not generalize to."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, performance differences could be driven by prompt format sensitivity, system message handling, or the specific LLM used as referee/judge, but none of these confounds are addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions are specified in Section 4.1.2: 'gpt-3.5-turbo-0613' for ChatGPT, 'gpt-4-0613' for GPT-4, 'claude-2' for Claude 2, and 'chat-bison-001' for PaLM."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts for all four evaluation tasks are provided in Appendix A.1, including initial prompts, round-by-round prompts, and judge prompts for each task. The actual text used is given, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported. Temperature, top-p, max tokens, and other sampling parameters for the LLM API calls are not specified. The public goods game uses alpha as a multiplier but LLM inference parameters are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding (message pool, referee, synchronous interaction algorithm) is described in detail in Section 3.3-3.4 with Algorithm 1. The workflow, message passing, and role assignment mechanisms are clearly documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Data preprocessing is poorly documented. For machine translation, the paper says 'We split the dataset into paragraph-level segments for the test set' but does not describe the splitting criteria. For idiom solitaire, 30 idioms are 'randomly sampled' from an unspecified database. How MBPP samples were selected is not described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations, threats to validity, or discussion section. The paper goes directly from experiments (Section 4) to conclusion (Section 5) without acknowledging any limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no mention of potential issues with using LLM-as-judge, prompt sensitivity, sample size limitations, or other threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No scope boundaries are stated. The paper does not explicitly state what settings or claims are excluded from its evaluation. The conclusion mentions future work but does not delineate what the current results do not show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw interaction logs, LLM outputs, and referee evaluations are not made available. Only aggregated scores are presented in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: LLMs interact through the framework with the synchronous algorithm (Algorithm 1), the referee collects and evaluates interaction histories. The process for each task is specified in Sections 3.7 and 4.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study evaluates LLMs only, using standard datasets (MBPP, IWSLT) and custom game scenarios."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw LLM interactions to final scores is not fully documented. For example, in code review and machine translation, the judge model scores are averaged but how invalid responses or parsing failures are handled is not described. The number of examples used from MBPP is not stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: all three authors are from the University of Science and Technology of China (School of Data Science and School of Computer Science). No affiliation with any LLM company being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of any funding disclosure is noted."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four models used. This is relevant because MBPP was published in 2021 and the models (especially GPT-4 and ChatGPT) may have seen it during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. MBPP is a well-known benchmark that could be in the training data of the models tested, but this is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MBPP was published in 2021 and is widely available online. The IWSLT 2017 dataset is also public. No discussion of whether these benchmarks could have been in the training data of the 2023-era models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It evaluates LLMs only."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs are reported despite the framework requiring multiple LLM API calls per evaluation (multi-round interactions for each task, repeated across multiple runs). The cost of running GPT-4, ChatGPT, Claude 2, and PaLM APIs is not mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The total API spend, number of API calls, or wall-clock time for the experiments is not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "The deep interaction-based framework can evaluate LLMs in dynamic real-world scenarios where static datasets cannot.",
       "evidence": "The paper demonstrates the framework on four tasks (public goods game, idiom solitaire, code review, machine translation) in Sections 4.2-4.5 with multi-round interactions between LLMs.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 has the state-of-the-art overall performance among the four tested LLMs.",
       "evidence": "GPT-4 ranks first in public goods game mode 1 (Figure 4a), code review (Table 3, highest programmer and reviewer scores), and DE-EN/EN-FR machine translation (Tables 4-5). However, ChatGPT outperforms GPT-4 in idiom solitaire (Table 1), EN-ZH translation (Table 6), and public goods game mode 2.",
       "supported": "moderate"
     },
     {
       "claim": "ChatGPT performs prominently in tasks related to Chinese.",
       "evidence": "ChatGPT has the highest winning rate in idiom solitaire (Table 1, average 0.77) and the best EN-ZH translation scores (Table 6). Section 4.5 notes consistency between idiom vocabulary and Chinese translation results.",
       "supported": "moderate"
     },
     {
       "claim": "The framework ensures fairness through anonymity and synchronous interaction.",
       "evidence": "The fairness condition (Condition 1) and synchronous interaction algorithm (Algorithm 1) are described in Sections 3.2-3.4. However, no empirical evidence is provided that these conditions actually affect evaluation outcomes or that violations would produce different results.",
       "supported": "weak"
     },
     {
       "claim": "The evaluation results converge with multiple repetitions (stableness condition).",
       "evidence": "The public goods game is run 10 times with boxplots showing variance in Figure 4. However, no formal convergence analysis is provided, and for the other three tasks, convergence is not explicitly demonstrated.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes DeepEval, a framework for evaluating LLMs through multi-round deep interactions rather than static benchmarks. Using game-theoretic principles, four evaluation tasks (public goods game, idiom solitaire, code review, machine translation) are designed to assess LLMs in interactive settings. Across these tasks, GPT-4 generally performs best overall while ChatGPT excels in Chinese-related tasks. The framework uses an LLM referee for scoring but does not validate these scores against human judgments or compare the framework's rankings against established evaluation methods.",
   "red_flags": [
     {
       "flag": "No validation against human evaluation or established benchmarks",
       "detail": "The paper proposes an evaluation framework but never validates whether its rankings align with human judgments or existing evaluation methods like MT-Bench, AlpacaEval, or standard code generation benchmarks. The LLM-as-judge referee is used without any inter-rater reliability assessment."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or discussion section. This is a significant omission for a paper proposing a new evaluation methodology, as there are many potential confounds (judge model bias, prompt sensitivity, small sample sizes)."
     },
     {
       "flag": "Circular evaluation concern",
       "detail": "The referee/judge is itself an LLM (unspecified which model). If GPT-4 is used as the judge to evaluate GPT-4 and its competitors, there is a potential self-preferencing bias. The paper does not state which model serves as referee or address this concern."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims about which model is 'state-of-the-art' are made based on raw score comparisons without any statistical tests. Score differences in Tables 3-6 are often very small (e.g., 8.97 vs. 8.85 vs. 8.86 in code review) and could easily be within noise."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the four LLMs, despite these parameters significantly affecting LLM output quality and consistency."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "MBPP (2021) is used for code review evaluation with 2023-era models that likely saw MBPP during training. This could inflate performance and affect the validity of the code review evaluation, but the paper does not acknowledge this risk."
     }
   ],
   "cited_papers": [
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used in the code review evaluation task, a key benchmark for LLM code generation evaluation."
     },
     {
       "title": "A Survey on Evaluation of Large Language Models",
       "authors": ["Yupeng Chang", "Xu Wang", "Jindong Wang"],
       "year": 2023,
       "arxiv_id": "2307.03109",
       "relevance": "Comprehensive survey on LLM evaluation methods, directly relevant to the survey's scope of assessing evaluation methodology quality."
     },
     {
       "title": "PaLM: Scaling Language Modeling with Pathways",
       "authors": ["Aakanksha Chowdhery", "Sharan Narang", "Jacob Devlin"],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "Describes PaLM, one of the four LLMs evaluated in the study, relevant to understanding LLM capability evaluation."
     },
     {
       "title": "AlpacaFarm: A Simulation Framework for Methods that Learn from Human Feedback",
       "authors": ["Yann Dubois", "Xuechen Li", "Rohan Taori"],
       "year": 2023,
       "arxiv_id": "2305.14387",
       "relevance": "LLM evaluation framework using simulation, relevant to automated LLM evaluation methodology."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the main models evaluated in LLM benchmark studies."
     },
     {
       "title": "GameEval: Evaluating LLMs on Conversational Games",
       "authors": ["Dan Qiao", "Chenfei Wu", "Yaobo Liang"],
       "year": 2023,
       "arxiv_id": "2308.10032",
       "relevance": "Uses conversational games for LLM evaluation, closely related approach to DeepEval's interaction-based evaluation."
     },
     {
       "title": "PandaLM: An Automatic Evaluation Benchmark for LLM Instruction Tuning Optimization",
       "authors": ["Yidong Wang", "Zhuohao Yu", "Zhengran Zeng"],
       "year": 2023,
       "arxiv_id": "2306.05087",
       "relevance": "Automated LLM evaluation using a discriminative language model, relevant to LLM-as-judge methodology."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "arxiv_id": "2306.05685",
       "relevance": "Foundational work on LLM-as-judge evaluation methodology, directly relevant to evaluating the validity of using LLMs as referees."
     },
     {
       "title": "PromptBench: Towards Evaluating the Robustness of Large Language Models on Adversarial Prompts",
       "authors": ["Kaijie Zhu", "Jindong Wang", "Jiaheng Zhou"],
       "year": 2023,
       "arxiv_id": "2306.04528",
       "relevance": "Evaluates adversarial robustness of LLMs, relevant to understanding LLM evaluation methodology and robustness benchmarks."
     },
     {
       "title": "Efficiently Measuring the Cognitive Ability of LLMs: An Adaptive Testing Perspective",
       "authors": ["Yan Zhuang", "Qi Liu", "Yuting Ning"],
       "year": 2023,
       "arxiv_id": "2306.10512",
       "relevance": "Adaptive testing framework for LLMs, directly relevant to efficient and dynamic LLM evaluation methodology."
     },
     {
       "title": "Synchromesh: Reliable Code Generation from Pre-trained Language Models",
       "authors": ["Gabriel Poesia", "Alex Polozov", "Vu Le"],
       "year": 2022,
       "relevance": "Addresses reliable code generation from LLMs, relevant to code generation evaluation."
     },
     {
       "title": "Natural Language to Code Generation in Interactive Data Science Notebooks",
       "authors": ["Pengcheng Yin", "Wen-Ding Li", "Kefan Xiao"],
       "year": 2023,
       "relevance": "Interactive code generation approach relevant to evaluating LLMs in real-world coding scenarios."
     }
   ]
 }