ai-research-survey

scan.json (29908B)

---

 {
   "paper": {
     "title": "HumanEval-XL: A Multilingual Code Generation Benchmark for Cross-lingual Natural Language Generalization",
     "authors": [
       "Qiwei Peng",
       "Yekun Chai",
       "Xuhong Li"
     ],
     "year": 2024,
     "venue": "International Conference on Language Resources and Evaluation",
     "arxiv_id": "2402.16694",
     "doi": "10.48550/arXiv.2402.16694"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "HumanEval-XL establishes a multilingual code generation benchmark spanning 23 natural languages and 12 programming languages with 22,080 prompts. GPT-4 consistently outperforms all other models across languages, while GPT-3.5 unexpectedly lags behind the code-specialized CodeGen2-16B on most PLs except Python. LLMs within the same family show high correlation across language resource levels (Pearson 0.8–0.87), but models from different families do not correlate. Low-resource and non-Latin-script languages (Afro-Asiatic, Greek, Iranian, Turkic families) consistently yield lower performance.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'We make our evaluation code and data publicly available at https://github.com/FloatAI/humaneval-xl.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (22,080 prompts across 23 NLs and 12 PLs) is released at the same GitHub URL mentioned in the abstract."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications (requirements.txt, Dockerfile, library versions) are provided in the paper. Only inference hyperparameters are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The stopping criteria and sampling parameters are described, but no runnable reproduction guide is provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The ± values in Tables 3 and 4 represent standard deviation across languages within a group, not uncertainty in the measurements. The main results in Tables 5–16 are point estimates with no confidence intervals or error bars on the pass@1 measurements."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The paper claims 'GPT-3.5 lags behind CodeGen2-16B' and makes other comparative claims based solely on comparing point estimates. Pearson correlations are reported (0.8, 0.87) but without significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1 scores are reported with sufficient context for comparison. For example, Table 3 shows GPT-4 at 78.54 vs GPT-3.5 at 62.50 on Class 5 Python, and all appendix tables provide per-model, per-language absolute scores enabling effect size computation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 80 parallel coding problems derived from Multilingual HumanEval. No justification is given for why 80 problems are sufficient for the claims being made across 23 NLs and 12 PLs."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs. The paper states 'Due to constrained computing resources, we report pass@1 for all experiments' but does not mention multiple runs or report variance across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple models are compared: CodeT5+ (220M, 770M, 2B), CodeGen2 (1B, 3.7B, 7B, 16B), GPT-3.5, and GPT-4, serving as baselines against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 (2023) and GPT-3.5 (2022) were contemporary at the time of writing. CodeGen2 and CodeT5+ were recent code-specialized models."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark construction has a multi-stage pipeline (extraction, translation, BERTScore quality check, manual quality control) but no ablation is provided to show the contribution of each stage (e.g., effect of removing BERTScore filtering)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only pass@1 is reported. The paper explicitly states: 'Due to constrained computing resources, we report pass@1 for all experiments.' No other metrics (pass@k, BLEU, etc.) are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluation was used only for quality control of the benchmark translations (Stage 4), not for evaluating the LLMs' code generation outputs. All system evaluation is automated via test case execution."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is used purely for evaluation with no model tuning. The translated prompts are new (created by GPT-4 translation), and models are evaluated zero-shot on these problems."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: per programming language (12 PLs, Tables 5–16), per natural language (23 NLs), per language resource class (Table 3), and per language family (Table 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative error analysis or failure case examples are provided. The paper reports aggregate pass@1 scores but does not examine what types of problems or prompts cause failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: CodeT5+ 'struggles to solve problems across most scenarios,' GPT-3.5 'lags behind CodeGen2-16B in all PLs except Python,' and 'current LLMs struggle to capture the equivalent semantic meaning expressed in different languages.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims are supported: the benchmark does cover 23 NLs and 12 PLs with 22,080 prompts and avg 8.33 test cases. The experiments confirm that LLMs struggle with cross-lingual NL generalization as claimed."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims without adequate justification: 'specialized code pre-training plays a pivotal role in code generation' and 'the encoder-decoder structure of CodeT5+ hinders its ability to complete code generation tasks.' These attribute performance differences to specific causes (architecture, training data) without controlled experiments isolating these factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper generally bounds its claims to the tested setting. The title references their specific benchmark, and claims like 'significant challenge for current LLMs' are framed within the context of their benchmark evaluation. The conclusion states it as 'a pioneering step.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. Performance differences are attributed to model architecture and training data without considering confounds like prompt format sensitivity, translation quality variation across languages, or GPT-4's advantage from having generated the translations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@1 on code generation tasks and frames results in terms of code generation capability. The measurement (pass@1 across NL-PL pairs) matches the claimed scope (multilingual code generation proficiency) without overreaching."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "GPT-3.5 and GPT-4 are referred to by marketing names only, with no API version or snapshot date (references cite OpenAI 2022 and OpenAI 2023 but no model version like gpt-4-0613). CodeT5+ and CodeGen2 specify parameter counts but not exact checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark problems themselves ARE the prompts (function signatures + NL docstrings), and these are released at the GitHub repository. The evaluation follows HumanEval's code completion format where the prompt is the problem description."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states: 'The specific parameters for the sampling process were set to a top-p value of 0.95 and a temperature of 0.2' applied consistently across all models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models perform direct code completion from prompts without any scaffold, tool use, or iterative refinement."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the 4-stage pipeline: NL extraction, GPT-4 translation with back-translation, BERTScore quality check (threshold 0.95, up to 3 iterations), and manual quality control with heuristic checks."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper goes directly from Analysis and Discussion (Section 4.3) to Conclusion (Section 5) without any substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. Issues like translation quality, small problem set size (80), single-run results, and potential contamination are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not articulate what the results do NOT show or what settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is released at https://github.com/FloatAI/humaneval-xl, allowing independent verification of the prompts, translations, and test cases."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the complete data construction process: starting from Multilingual HumanEval, extracting NL portions, translating via GPT-4, back-translating for quality checking with BERTScore, and manual verification."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The benchmark is derived from the existing Multilingual HumanEval dataset through automated translation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section 3.2 and Figure 1: extraction → translation → back-translation → BERTScore filtering (threshold 0.95, up to 3 iterations) → manual quality control. The final count of 80 parallel problems is stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work was partially supported by DisAI... a project funded by European Union under the Horizon Europe, GA No. 101079164.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Qiwei Peng from University of Copenhagen, Yekun Chai and Xuhong Li from Baidu Inc."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "EU Horizon Europe funding for combating disinformation is independent of the benchmark evaluation outcomes. However, two authors are from Baidu, which develops competing LLMs, though no Baidu models are evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Two authors are affiliated with Baidu Inc., which has commercial interests in LLM development."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models (GPT-3.5, GPT-4, CodeGen2, CodeT5+). This is critical since the benchmark derives from HumanEval (published 2021)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The benchmark problems derive from HumanEval, which was published in 2021 and is widely available online. Models trained after 2021 may have seen the original problems."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval has been publicly available since 2021. All evaluated models (GPT-3.5, GPT-4, CodeGen2, CodeT5+) were likely trained on data including HumanEval solutions. While the NL prompts are translated, the underlying coding problems and test cases are the same. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLMs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs or latency are reported. The paper mentions 'constrained computing resources' as motivation for only reporting pass@1 but does not quantify the actual cost of running 22,080 evaluations across 9 model configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. Neither GPU hours for open-source models nor API costs for GPT-3.5/GPT-4 evaluations are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results appear to be from single runs with temperature=0.2 and top-p=0.95. No variation across random seeds is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. The results in Tables 5–16 appear to be from single evaluations."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The choice of temperature=0.2 and top-p=0.95 is stated but not justified. No hyperparameter search is mentioned."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sampling parameters (temperature=0.2, top-p=0.95) are presented without justification for why these values were chosen or how they affect results."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across 9 model configurations, 23 NLs, and 12 PLs without any multiple comparison correction. Pearson correlations are reported without significance testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluate models on it without acknowledging the bias of evaluating on their own benchmark. Additionally, GPT-4 was used to create the translations and is also one of the evaluated models."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Performance is not reported as a function of compute budget. Models of vastly different sizes and costs are compared on pass@1 without normalizing for compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether pass@1 on GPT-4-translated prompts actually measures 'cross-lingual NL generalization' as claimed, or whether translation artifacts could confound the results. The BERTScore quality check ensures translation similarity but does not validate the construct."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All models perform direct code completion without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. HumanEval was published in 2021. All evaluated models (GPT-3.5/4, CodeGen2, CodeT5+) were trained on data potentially including HumanEval and its solutions. The translated NL prompts are new, but the underlying coding problems and test cases are identical."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The function signatures and test cases remain unchanged from HumanEval, meaning models that memorized HumanEval solutions could potentially benefit regardless of the NL prompt language."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. All 22,080 prompts derive from the same 80 HumanEval problems (translating each into 23 NLs × 12 PLs), meaning results across NLs for the same problem are not independent."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection method is applied. No canary strings, membership inference, n-gram overlap analysis, or temporal splits are used."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 consistently demonstrates superior performance across different PLs and NLs (except C#), outperforming all other models significantly.",
       "evidence": "Figure 2 and Tables 5–16 show GPT-4 achieving the highest pass@1 scores across nearly all PL-NL combinations, e.g., 82.50% on English-Python vs 66.25% for GPT-3.5 (Table 5).",
       "supported": "strong"
     },
     {
       "claim": "GPT-3.5 lags behind CodeGen2-16B in all PLs except Python, despite having considerably more parameters, underscoring the crucial role of training on coding data.",
       "evidence": "Tables 5–16 show CodeGen2-16B outperforming GPT-3.5 on Java, JavaScript, C#, Go, Kotlin, PHP, Ruby, Scala, Swift, and TypeScript. The causal attribution to 'coding data' is speculative.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs within the same family but with different parameter scales exhibit high correlation across different levels of language resources (Pearson 0.8 for CodeGen2, 0.87 for GPT).",
       "evidence": "Section 4.3 reports Pearson correlations computed from Table 3 data, comparing performance patterns across NL resource classes.",
       "supported": "moderate"
     },
     {
       "claim": "The encoder-decoder architecture of CodeT5+ hinders its ability to complete code generation tasks compared to decoder-only models.",
       "evidence": "Tables 5–16 show CodeT5+ (2B) performing near-zero across most scenarios while CodeGen2 (3.7B) with a decoder-only structure performs much better. However, the causal attribution to architecture alone ignores differences in training data and scale.",
       "supported": "weak"
     },
     {
       "claim": "Languages from Afro-Asiatic, Indo-European (Greek), Indo-European (Iranian), and Turkic families generally yield lower results compared to other language families.",
       "evidence": "Table 4 shows these language families scoring lower, e.g., Greek yields 71.25% for GPT-4 vs 80.31% for Indo-European (Germanic) on Python.",
       "supported": "moderate"
     },
     {
       "claim": "Current LLMs struggle to capture the equivalent semantic meaning expressed in different languages for code generation.",
       "evidence": "Performance variation across NLs is shown in Figure 2 and Tables 3–4 with std dev across language groups (e.g., GPT-3.5 Python: 62.50±5.06 for Class 5 vs 60.42±2.86 for Class 3). The variation is modest for GPT-4 (~77-79% across groups).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "GPT-4 translation circularity",
       "detail": "GPT-4 was used to translate the benchmark prompts into 23 languages (Section 3.2) AND is one of the evaluated models. GPT-4 may have an advantage on prompts it generated, creating a systematic bias in its favor. This confound is never discussed."
     },
     {
       "flag": "No contamination analysis on HumanEval-derived problems",
       "detail": "The benchmark's 80 coding problems are directly derived from HumanEval (published 2021). All evaluated models were likely trained on data containing HumanEval solutions. While prompts are translated, the function signatures and test cases remain identical, allowing memorized solutions to still work. This fundamental contamination risk is completely ignored."
     },
     {
       "flag": "Single metric with single runs",
       "detail": "Only pass@1 is reported from apparently single runs. With only 80 problems per NL-PL pair, individual pass@1 values have a granularity of 1.25 percentage points (1/80), meaning small reported differences may not be meaningful."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats to validity, or scope boundary discussion despite significant methodological concerns (small problem set, contamination risk, translation quality, single metric)."
     },
     {
       "flag": "Unjustified causal attributions",
       "detail": "Performance differences are attributed to specific causes (model architecture, code-specific training) without controlled experiments. Multiple confounds exist: training data size and composition, instruction tuning, RLHF, and model scale all differ simultaneously."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduced HumanEval, the foundational benchmark from which HumanEval-XL is derived; central to LLM code generation evaluation."
     },
     {
       "title": "Multi-lingual Evaluation of Code Generation Models",
       "authors": ["Ben Athiwaratkun", "Sanjay Krishna Gouda", "Zijian Wang"],
       "year": 2023,
       "relevance": "Extended HumanEval to multiple programming languages (Multilingual HumanEval), the direct predecessor to HumanEval-XL."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark for code generation evaluation, one of the key benchmarks in this space."
     },
     {
       "title": "ERNIE-Code: Beyond English-Centric Cross-Lingual Pretraining for Programming Languages",
       "authors": ["Yekun Chai", "Shuohuan Wang", "Chao Pang"],
       "year": 2023,
       "relevance": "Pioneered multilingual NL+PL pretraining across 116 NLs and 6 PLs, directly relevant to cross-lingual code generation."
     },
     {
       "title": "CodeGen2: Lessons for Training LLMs on Programming and Natural Languages",
       "authors": ["Erik Nijkamp", "Hiroaki Hayashi", "Caiming Xiong"],
       "year": 2023,
       "arxiv_id": "2305.02309",
       "relevance": "One of the primary models evaluated in this paper; code-specialized LLM for multilingual code generation."
     },
     {
       "title": "CodeT5+: Open Code Large Language Models for Code Understanding and Generation",
       "authors": ["Yue Wang", "Hung Le", "Akhilesh Deepak Gotmare"],
       "year": 2023,
       "arxiv_id": "2305.07922",
       "relevance": "Encoder-decoder code LLM evaluated in the paper, provides architecture comparison for code generation."
     },
     {
       "title": "Execution-Based Evaluation for Open-Domain Code Generation",
       "authors": ["Zhiruo Wang", "Shuyan Zhou", "Daniel Fried", "Graham Neubig"],
       "year": 2022,
       "arxiv_id": "2212.10481",
       "relevance": "Introduced ODEX dataset with 4 NLs and Python, a direct multilingual code generation predecessor."
     },
     {
       "title": "Starcoder: May the Source Be With You!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Major open-source code LLM relevant to multilingual code generation capabilities."
     },
     {
       "title": "CodeGeeX: A Pre-trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X",
       "authors": ["Qinkai Zheng", "Xiao Xia", "Xu Zou"],
       "year": 2023,
       "relevance": "Extended HumanEval to multiple PLs (HumanEval-X), directly related benchmark for multilingual code generation."
     },
     {
       "title": "BLOOM: A 176B-Parameter Open-Access Multilingual Language Model",
       "authors": ["Teven Le Scao", "Angela Fan", "Christopher Akiki"],
       "year": 2022,
       "arxiv_id": "2211.05100",
       "relevance": "Large multilingual model pre-trained on 46 NLs and 13 PLs, relevant to multilingual code generation evaluation."
     },
     {
       "title": "Measuring Coding Challenge Competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Code generation benchmark with 10,000 problems from competitions, part of the evaluation landscape HumanEval-XL extends."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The benchmark is directly usable by practitioners evaluating multilingual code generation models; code and data are released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that LLMs struggle with non-English prompts is somewhat expected; the GPT-3.5 vs CodeGen2 finding is mildly surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns are raised by this benchmark paper."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward benchmark contribution."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data are publicly available at the GitHub URL for immediate use and evaluation."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates GPT-4 and GPT-3.5 which are well-known, but the benchmark itself is from less prominent authors (Copenhagen/Baidu)."
     }
   }
 }