ai-research-survey

scan.json (30186B)

---

 {
   "paper": {
     "title": "Importance Sampling is All You Need: Predict LLM's performance on new benchmark by reusing existing benchmark",
     "authors": [
       "Junjie Shi",
       "Wei Ma",
       "Shi Ying",
       "Lingxiao Jiang",
       "Yang Liu",
       "Bo Du"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2508.01203",
     "doi": "10.48550/arXiv.2508.01203"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "BIS proposes a prompt-centric framework using importance sampling and IWAE to predict LLM performance on code generation benchmarks without ground-truth execution, achieving ~1.1% average absolute error on CodeBLEU predictions across CodeLlama models (7B–70B). The method outperforms alternative distribution modeling approaches (GMM, RBM, MaxEnt, VAE) and regression baselines (MLP, RNN, LR, etc.) in cross-benchmark prediction between BigCodeBench and EvoEval. Semantic-level metrics (security score, cyclomatic complexity) are more accurately predicted than code-level metrics (Halstead complexity), with errors of 4.3% vs 10.7% at worst.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: BigCodeBench, HumanEval, and EvoEval (7 sub-benchmarks). All are standard public benchmarks that are freely accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions renting '8 server with L20 GPUs' (Section 4.1) but provides no requirements.txt, Dockerfile, library versions, or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results tables (Tables 3–13) report only point estimates of error. No confidence intervals, error bars, or ± notation appear anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'our approach achieves the lowest prediction error among all compared schemes' (Section 4.4) based solely on comparing raw numbers across Tables 7–8. No p-values, t-tests, or any statistical significance tests are used."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Error rates are reported with full baseline context. For example, BIS achieves avg absolute error 0.011 vs GMM(80) at 0.290 and RNN at 0.298 (Tables 7–8), providing clear magnitude comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses ~8,528 data points across 4 models and 9 benchmarks but provides no justification for why this is sufficient, no power analysis, and no discussion of statistical adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across experimental runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are compared: distribution modeling methods (GMM, RBM, MaxEnt, VAE) in Table 7, and regression methods (RSR, LR, DTR, RR, MLP, RNN) in Table 8."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include VAE (a direct contemporary alternative) and standard ML/DL approaches. For the specific task of prompt distribution modeling, these represent reasonable and appropriate comparisons."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 (RQ4) presents a thorough ablation studying four factors: feature dimensionality (Tables 9–10), IWAE sample count (Table 11), weight truncation percentile (Table 12), and prompt set size (Table 13)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics evaluated: CodeBLEU, pass@1, security score, cyclomatic complexity, and four Halstead metrics (length, volume, effort, time) in Table 6."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of any kind. All evaluation is automated comparison of predicted vs actual metric values."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The framework uses cross-prediction between entirely separate benchmark suites: BigCode predicts Evo and vice versa (Section 4.1). Source and target are disjoint datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model size (7B, 13B, 34B, 70B), by source/target direction, and by metric type (semantic vs code-level) across all tables."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows general degradation trends (e.g., small prompt sets, extreme truncation) but does not analyze specific failure cases or show qualitative examples where the framework produced particularly poor predictions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results reported: code-level metrics perform much worse than semantic metrics (Table 6, up to 10.7% error), linear layer dimensionality reduction severely degrades performance (Table 10, errors exceeding 0.4), and small prompt sets cause large errors (Table 13)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '1.1% average absolute prediction error' are supported by Tables 3 (0.8% and 1.4% average absolute errors), '0.3% and 1.9%' best/worst are visible in Table 3, and '2.15% for pass@1' matches Table 4 (2.2% and 2.1%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The core causal claim—that prompt distributions determine LLM performance—is supported by a mathematical proof (Theorem 3.1, Section 3.3). Ablation studies in Section 4.5 use controlled single-variable manipulation to support component-level causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Importance Sampling is All You Need' and abstract claims of 'broad applicability' are unbounded. The paper tests only CodeLlama models (one family) on code generation tasks in Python. Section 5 acknowledges cross-scenario and cross-language as future work but the title and abstract do not reflect these scope limitations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for its results. For example, the low prediction errors could be due to high distributional similarity between BigCodeBench and EvoEval (both Python code generation) rather than the method's generalizability. No robustness checks against confounds are performed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "CodeBLEU measures syntactic/structural similarity to reference solutions, not actual code correctness. The paper uses CodeBLEU as its primary 'correctness metric' (Section 4.2: 'To represent correctness of code generated by LLM, we employ the CodeBLEU score') without discussing the gap between reference similarity and actual functional correctness. Pass@1 (which measures execution correctness) is only tested on a single model."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'CodeLlama-7B, CodeLlama-13B, CodeLlama-34B, and CodeLlama-70B' (Section 4.1), with citation to the Code Llama paper [27]. These are unique, identifiable model releases."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation prompts come from publicly available benchmarks (BigCodeBench, HumanEval, EvoEval) which provide their prompts openly. The paper's method operates on these benchmark prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "IWAE hyperparameters are studied in the ablation (K=10 samples, 0.9 truncation percentile, 768-dim embeddings). However, LLM generation hyperparameters (temperature, top-p, max tokens) for the CodeLlama runs are never reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The framework consists of BERT embeddings, IWAE models, and importance weight computation—no agent loops or tool use."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents preprocessing: merging HumanEval with EvoEval into 'Evo' dataset (992 samples), BigCodeBench as standalone (1,140 samples), min-max normalization to [0,1], BERT CLS token extraction for embeddings."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Discussion' provides substantive discussion of limitations including cross-scenario/cross-language applicability, Assumption 1 violations, and weight instability issues."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 and Section 3.3.1 discuss specific threats: Assumption 1 violation when source and target distributions don't overlap, extreme weight distributions dominating predictions, and that cross-domain settings increase weight variance."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. Section 5 frames scope limitations as 'future work' directions rather than explicit boundaries. No statement like 'our results do not generalize to non-code tasks' or 'results are limited to the CodeLlama family.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (model outputs, computed scores, embeddings, importance weights) is released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes: 8,528 data points collected across 4 CodeLlama models and 9 benchmarks, using 8 servers with L20 GPUs. Table 2 provides per-benchmark data point counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from running CodeLlama models on standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from model outputs to final predictions is described at a high level (Figure 1), but critical details are missing: how many generations per prompt, inference settings for CodeLlama, exact IWAE training procedure, and how CodeBLEU reference solutions were obtained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7: 'This research is supported by the Ministry of Education, Singapore under its Academic Research Fund Tier 3 (Award ID: MOET32020-0004).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Nanyang Technological University, Singapore Management University, and Wuhan University. No affiliation with Meta (CodeLlama developer) or benchmark creators."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Ministry of Education, Singapore has no financial interest in the outcome of CodeLlama benchmarking research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit training data cutoff date for CodeLlama is stated. The paper says 'we employed open-source models released before the publication of these benchmarks' (Section 4.1) but does not specify when training data collection ended."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims models were 'released before the publication of these benchmarks' but this is inaccurate for HumanEval (published 2021, CodeLlama released 2023). The paper does not analyze whether CodeLlama's training data included HumanEval problems."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) was publicly available well before CodeLlama's training. The paper's claim that models predate benchmarks is incorrect for HumanEval. No contamination analysis is performed despite this being a core motivation of the paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'we rented 8 server with L20 GPUs with the bill 280$.' This states the total compute cost for the experiments."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: '8 server with L20 GPUs with the bill 280$,' providing the total hardware and cost budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis for IWAE training or model inference. For pass@1, CodeLlama-7B was run 10 times but this is inherent to the metric definition, not a sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated for the main experiments (CodeBLEU predictions). Only pass@1 states 'ten times' (Section 4.2), but this is metric-specific, not a replication count."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The ablation study (Section 4.5) explores different hyperparameter settings but does not report the total search budget or how many configurations were evaluated before selecting the final settings."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 justifies configuration selection: K=10 IWAE samples selected based on Table 11 (avoiding both too-few and too-many extremes), truncation at 0.9 percentile based on Table 12, full 768-dim embeddings based on Tables 9–10."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 8+ baselines, 4 models, and 2 dataset directions without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baseline methods themselves but do not acknowledge the potential bias of author-implemented baselines systematically underperforming."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different methods (IWAE vs GMM vs RNN vs MLP) have different compute requirements, but performance is never reported as a function of compute. The comparison is accuracy-only with no compute normalization."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "CodeBLEU is used as the primary 'correctness' metric but it measures reference similarity, not functional correctness. The paper does not discuss whether CodeBLEU validly measures code correctness, nor whether predicting CodeBLEU scores has practical value for benchmark evaluation."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the evaluation. Models generate code directly from prompts."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (published 2021) predates CodeLlama's training (2023). The paper claims models predate benchmarks but this is incorrect for HumanEval. Temporal leakage is not analyzed per-benchmark."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, whether BERT embeddings of prompts could encode benchmark-specific signals that inflate prediction accuracy."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "EvoEval is explicitly derived from HumanEval (evolved versions of HumanEval problems). The paper merges them into one 'Evo' dataset and uses it for cross-prediction with BigCodeBench, but does not discuss whether this shared origin could inflate distributional similarity or affect results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied despite the paper's stated motivation of addressing data contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "BIS achieves an average absolute prediction error of 1.1% for CodeBLEU code correctness scores across 4 CodeLlama models.",
       "evidence": "Table 3 (Section 4.2) shows average absolute errors of 0.8% (BigCode→Evo) and 1.4% (Evo→BigCode), averaging approximately 1.1%.",
       "supported": "moderate"
     },
     {
       "claim": "BIS generalizes to pass@1 with average absolute errors of 2.15%.",
       "evidence": "Table 4 (Section 4.2) shows errors of 2.2% and 2.1% for the two prediction directions, but only on CodeLlama-7B.",
       "supported": "weak"
     },
     {
       "claim": "BIS outperforms all baseline distribution modeling and regression methods.",
       "evidence": "Tables 7–8 (Sections 4.3–4.4) show BIS achieving 0.011 avg absolute error vs next best VAE (0.015), MaxEnt (0.017), and RBM (0.018). Non-IS baselines: LR/RR (0.017), RSR (0.031), MLP (0.105), RNN (0.298).",
       "supported": "moderate"
     },
     {
       "claim": "Under fixed model and evaluation metrics, prompt distribution uniquely determines expected model performance.",
       "evidence": "Theorem 3.1 (Section 3.3) provides a mathematical proof using importance sampling theory, with Assumption 1 (absolute continuity) as the key requirement.",
       "supported": "strong"
     },
     {
       "claim": "Semantic-level metrics are more accurately predicted than code-level metrics.",
       "evidence": "Table 6 (Section 4.3) shows max absolute errors of 4.3% and 4.6% for security score and cyclomatic complexity, versus 10.7%, 8%, 5%, and 4.5% for Halstead metrics.",
       "supported": "moderate"
     },
     {
       "claim": "BIS can reduce the cost and effort of benchmarking LLMs by eliminating need for ground-truth test suites.",
       "evidence": "The framework predicts performance from prompt distributions alone (Section 3.2). However, this is only validated on two benchmark groups using a single model family, and relies on having a fully annotated source benchmark.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Extremely narrow model diversity",
       "detail": "All 4 models are from the CodeLlama family, sharing a common lineage (acknowledged in Section 4.1: 'models within this family share a common lineage'). Results cannot demonstrate generalization across architecturally diverse models. No instruction-tuned, chat, or closed-source models are tested."
     },
     {
       "flag": "No code or data released",
       "detail": "Despite proposing a novel framework, no source code, trained models, or raw experimental data is released. The results cannot be independently verified or reproduced."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Claims of outperforming baselines are based entirely on comparing point estimates without any significance tests, confidence intervals, or variance measures. The margins between BIS (0.011) and VAE (0.015) or MaxEnt (0.017) are small and could be within noise."
     },
     {
       "flag": "Contamination contradiction",
       "detail": "The paper motivates itself partly by addressing data contamination risk, yet does not analyze contamination in its own experiments. HumanEval (2021) was public before CodeLlama's training, creating a contamination risk that goes unaddressed."
     },
     {
       "flag": "CodeBLEU as 'correctness' proxy",
       "detail": "The primary metric CodeBLEU measures syntactic/structural similarity to reference solutions, not functional correctness. The paper calls it a 'correctness metric' (Section 4.2) without discussing this validity gap. Pass@1 (actual execution correctness) is only tested on a single model."
     },
     {
       "flag": "EvoEval derived from HumanEval non-independence",
       "detail": "EvoEval benchmarks are evolved from HumanEval problems (ref [36]). Merging them into one 'Evo' dataset and cross-predicting with BigCodeBench may benefit from shared distributional characteristics that wouldn't exist with truly independent benchmarks."
     },
     {
       "flag": "Overclaiming title and abstract",
       "detail": "The title 'Importance Sampling is All You Need' and abstract claims of 'broad applicability' significantly overstate the scope of experiments limited to one model family on Python code generation tasks."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, one of the most widely used LLM code generation benchmarks."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo", "Minh Chien Vu"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Major code generation benchmark requiring complex function calls, used as primary evaluation dataset in this paper."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM family used as the sole evaluation models in this paper."
     },
     {
       "title": "Top leaderboard ranking = top coding proficiency, always? EvoEval: Evolving coding benchmarks via LLM",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2403.19114",
       "relevance": "Creates evolved benchmarks from HumanEval to test robustness of code generation evaluation."
     },
     {
       "title": "SWE-Dev: Evaluating and Training Autonomous Feature-Driven Software Development",
       "authors": ["Yaxin Du", "Yuzhu Cai"],
       "year": 2025,
       "arxiv_id": "2505.16975",
       "relevance": "Code generation benchmark for realistic software engineering settings."
     },
     {
       "title": "Swe-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Influential benchmark for evaluating LLMs on real-world software engineering tasks."
     },
     {
       "title": "A survey on evaluating large language models in code generation tasks",
       "authors": ["Liguo Chen", "Qi Guo"],
       "year": 2024,
       "arxiv_id": "2408.16498",
       "relevance": "Comprehensive survey of LLM code generation evaluation methodologies."
     },
     {
       "title": "TestGenEval: A real world unit test generation and test completion benchmark",
       "authors": ["Kush Jain", "Gabriel Synnaeve", "Baptiste Rozière"],
       "year": 2024,
       "arxiv_id": "2410.00752",
       "relevance": "Benchmark for automated test generation showing only 39.2% accuracy, motivating the need for cheaper evaluation methods."
     },
     {
       "title": "LessLeakBench: A First Investigation of Data Leakage in LLMs Across 83 Software Engineering Benchmarks",
       "authors": ["Xin Zhou", "Martin Weyssow"],
       "year": 2025,
       "arxiv_id": "2502.06215",
       "relevance": "Investigates data leakage across SE benchmarks, directly relevant to the contamination concerns this paper claims to address."
     },
     {
       "title": "Does Data Contamination Detection Work (Well) for LLMs? A Survey and Evaluation on Detection Assumptions",
       "authors": ["Yujuan Fu", "Ozlem Uzuner"],
       "year": 2024,
       "arxiv_id": "2410.18966",
       "relevance": "Surveys data contamination detection methods for LLMs, relevant to the contamination mitigation claims of BIS."
     },
     {
       "title": "CodeBLEU: a Method for Automatic Evaluation of Code Synthesis",
       "authors": ["Shuo Ren", "Daya Guo"],
       "year": 2020,
       "arxiv_id": "2009.10297",
       "relevance": "Primary evaluation metric used in this paper for measuring code generation quality."
     },
     {
       "title": "Task contamination: Language models may not be few-shot anymore",
       "authors": ["Changmao Li", "Jeffrey Flanigan"],
       "year": 2024,
       "relevance": "Demonstrates that contaminated benchmark samples can inflate LLM scores by up to 5x, supporting the motivation for contamination-resistant evaluation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The framework could help benchmark developers screen candidate tasks, but requires training IWAE models and having a fully annotated source benchmark, limiting immediate practical use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Predicting performance without execution is a novel framing, but the underlying idea that similar prompts yield similar performance is fairly intuitive."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised by this work."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Touches on the data contamination debate in benchmarks, which is a contentious topic, but does not make dramatic claims about specific benchmarks or models being compromised."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors from NTU, SMU, and Wuhan University; not about well-known products or from prominent AI labs."
     }
   }
 }