ai-research-survey

scan.json (35006B)

---

 {
   "paper": {
     "title": "Narrowing the Complexity Gap in the Evaluation of Large Language Models",
     "authors": [
       "Yang Chen",
       "Shuyang Liu",
       "Reyhaneh Jabbarvand"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.18928"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GeneBench uses multi-objective genetic optimization to transform 4 existing benchmarks (HumanEval, ClassEval, CRUXEval, Avatar) into more complex versions, increasing relative complexity by 298% on average while maintaining readability (12% decrease). Evaluating 13 LLMs across 4 programming tasks shows a 14.9%–60.5% (avg 35.2%) performance drop, with the struggle persisting under few-shot prompting (avg 41% drop) and fine-tuning (avg 17% gap). Performance on GeneBench bug repair correlates with SWE-Bench performance, suggesting it can proxy real-world benchmarks without costly construction.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or archive is provided in the paper. GeneBench is described but its implementation is not released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The input benchmarks (HumanEval, ClassEval, CRUXEval, Avatar) are public, but GeneBench's transformed benchmark outputs are not released. No download link for the generated benchmarks is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions A100 GPUs, bfloat16 precision, Python AST library, and py2cfg, but provides no requirements.txt, Dockerfile, or detailed dependency listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The algorithm is described at a high level (Algorithms 1-2) but not with enough specificity to reproduce without the source code."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 reports point estimates for all models and tasks with no confidence intervals, error bars, or ± notation. Figures report point values without uncertainty bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Statistical significance tests are reported: p-value = 6e-18 for overall performance drop (§5.3), p-value = 4e-8 for complexity distribution difference between Success_Success and Success_Failure groups (§5.3.4), and Spearman's rank correlation with p-value for operator frequency vs complexity (§5.2)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentage drops with baseline context throughout: Δ = (Aft−Bef)/Bef (Table 5), e.g., 14.9%–60.5% avg=35.2% drop. Relative complexity increases (75%–650%) and readability decreases (10%–15%) are also provided in Table 4."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for sample sizes. CRUXEval is sampled at 200 of 800 programs without justification. SWE-Sub filters to 94 instances without discussing whether this is sufficient for the claims. No rationale for the 13-model selection."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Table 5) are single-run at temperature=0 with no variance or spread measures. The single-operator experiment was repeated 3 times, but only qualitative summary ('almost the same outcome') is given, not quantitative variance. Three GeneBench versions (§5.4.3) show consistency but without formal spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Before-transformation performance serves as the baseline across all experiments (Bef. columns in Table 5). SWE-Bench performance is used as a real-world baseline for comparison in §5.5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Evaluated models include recent ones: GPT-4o, o4-mini, DeepSeek-R1 (2024–2025 models). The comparison framework uses SWE-Bench, a contemporary real-world benchmark. Prior transformation techniques are discussed and compared (Table 3, §4)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Single-operator experiments (§5.3) test each of 22 operators independently. Figure 8 shows per-operator frequency and complexity contribution. §5.3.4 isolates the effect of complexity vs representation shift through NNMR analysis."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: success rate per task, relative complexity (RC), relative readability (RR), PyLint score, NNMR, CodeBLEU, cosine similarity, Jaccard similarity, and Spearman's rank correlation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of GeneBench transformations, their naturalness, or LLM outputs. The paper relies entirely on automated metrics (test pass/fail, PyLint scores, readability metrics) to assess quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For fine-tuning (§5.4.2), training uses transformations NOT selected for the final benchmark, with deduplication (Jaccard > 0.8). The benchmark transformations (from the Pareto front) serve as the held-out test set. For the main experiment, temperature=0 means no tuning on test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides per-model and per-task breakdowns across 5 benchmark-task combinations and 13 models. Figure 8 breaks down per-operator contributions. Figure 9 shows per-task Success_Success and Success_Failure distributions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§5.3 includes extensive failure analysis: §5.3.1 analyzes across tasks, §5.3.2 across models, §5.3.3 by size/training strategy, §5.3.4 investigates root causes of failure (representation shift vs complexity). Figure 10 compares complexity of success vs failure cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that NNMR (representation shift) does NOT correlate with performance drop (§5.3.4), ruling out a plausible explanation. Few-shot learning sometimes worsens performance (green vs red in Table 5 Δ(IC)). Fine-tuning leaves substantial performance gaps (Figure 11)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 14.9%–60.5% avg=35.2% performance drop (Table 5), struggle persists under few-shot (Table 5 Δ(IC)) and fine-tuning (Figure 11, 4.7%–30.3% avg=17%), SWE-Bench similarity shown in §5.5. The claim of 'consistent reproduction' is supported by three versions (§5.4.3)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims complexity increases cause performance drops. The study design controls for this: same programs before/after transformation, single-operator experiments confirm combination matters, NNMR analysis rules out representation shift as the sole cause, and Figure 10 shows statistically significant complexity differences between success/failure groups (p=4e-8)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Narrowing the Complexity Gap in the Evaluation of Large Language Models' is broad, but results are exclusively on Python. §7 acknowledges 'Our current implementation targets Python' but frames multi-language as 'primarily an engineering effort,' understating the potential for different results in other languages."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§5.3.4 systematically investigates alternative explanations: representation shift (NNMR, Table 6), increased input length (token counts vs context windows), and data contamination (multiple GeneBench versions). They rule out representation shift as the sole driver and show complexity is the primary factor (Figure 10)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly tests whether GeneBench performance proxies real-world performance through the SWE-Bench comparison (§5.5), showing GeneBench repair rates (45.3%/20.6%) are closer to SWE-Sub (17.6%) than original benchmarks (66.4%/32.1%). They acknowledge this validation is limited to repair, noting no real-world benchmarks exist for code reasoning."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models are identified by size (e.g., 'CodeLlama-13B-Base', 'DeepSeekCoder-6.7B-Instruct') but proprietary models GPT-4o and o4-mini lack specific API versions or snapshot dates. References cite generic URLs (e.g., [1] links to GPT-4 technical report, [5] links to o4-mini docs page) without version specifics."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 6 shows prompt templates with placeholders (${CODE}, ${TESTS}, ${EXAMPLE_CODE}, ${EXAMPLE CoT REASONING}). While the benchmark data (fill values for code/tests) is from public sources, the specific ICL example selections, CoT reasoning examples, and exact prompt text sent to models are not fully specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature=0 is reported for inference (§5.1.1). 'For other parameters, we used the default settings' without specifying what those defaults are. Fine-tuning uses QLoRA on 8 A100-40GB GPUs for up to 2 epochs with early stopping, but learning rate, LoRA rank, alpha, batch size, and other training hyperparameters are not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "GeneBench is a program transformation tool, not an agentic scaffold. The Agentless tool used in §5.5 is a third-party black-box repair tool — the authors cannot be expected to describe its internal scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented: CRUXEval sampling (200 programs), SWE-Bench-Lite filtering to SWE-Sub (300 → 94 instances with criterion: >3 lines additions vs deletions), fine-tuning deduplication (Jaccard > 0.8, resulting in 106k samples), test coverage validation (90.8%–99.5% original, 93.8%–99.4% transformed)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§7 'Threats to Validity' provides a dedicated section with three subsections: External Validity, Internal Validity, and Construct Validity, each discussing specific concerns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: Python-only implementation (external), temperature=0 and bfloat16 precision choices (internal), reliance on test execution for semantic equivalence with coverage statistics (internal), and use of Python AST and py2cfg tools (construct)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§7 explicitly states 'Our current implementation targets Python, as it dominates benchmarks like HumanEval and SWE-Bench.' The paper also notes GeneBench is for code-to-code/text tasks (footnote 2 states it is 'orthogonal to research on automated generation of program synthesis (text-to-code) problems')."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data is available. Neither the generated GeneBench transformations, the LLM outputs, nor the detailed per-problem results are released. Only aggregated results in tables and figures are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is well-described: 4 benchmarks selected with rationale (§5.1.2), 200 CRUXEval programs sampled, SWE-Sub filtered from SWE-Bench-Lite (94 of 300), operator design based on manual examination of 2,692 Python files from top 100 PyPI projects (§4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (HumanEval, ClassEval, CRUXEval, Avatar, SWE-Bench)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: Algorithms 1-2 describe the transformation and validation process, including selection (roulette wheel + NSGA-II), transformation, readability/PyLint validation, test execution, and Pareto front selection. SWE-Sub filtering (300→94) and fine-tuning deduplication (→106k samples) include counts and criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments, grants, or sponsorship information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors' affiliations are listed as University of Illinois Urbana-Champaign. The paper evaluates third-party models (GPT-4o, DeepSeek, CodeLlama, etc.) without affiliation to any of the model providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. University researchers typically receive grant funding, but none is mentioned."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes 'more than 90% of SWE-Bench problems are dated before the training cut-off date of most recent LLMs' (§1) but does not state the actual training cutoff dates for any of the 13 evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Data contamination is extensively discussed as a core motivation (L3, §1). The paper argues GeneBench 'overcomes L3 limitation by upgrading programs to more complex versions with several non-trivial changes.' §5.4.3 shows different GeneBench versions produce different programs (Figure 12), and Table 6 shows high NNMR (19.4%–97.65%) indicating transformed versions are embedding-distant from originals."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is a primary concern addressed throughout. The paper explicitly notes original benchmarks (HumanEval, 2021) predate most models' training data. GeneBench's entire purpose is to mitigate this by creating novel transformed versions that are 'generated through the non-deterministic application of 22 genetic operators; re-executing GeneBench results in different programs' (§1)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study using automated metrics."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on transformed programming benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example latency are reported for the LLM evaluations. Token counts are listed in Table 6 but not translated into cost or time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Partial: GeneBench uses a 1-hour time budget per program (§5.1), and fine-tuning uses 8 A100-40GB GPUs for up to 2 epochs. But total GPU-hours for all experiments, API costs for GPT-4o/o4-mini, and total compute for the full evaluation pipeline are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main LLM evaluations use temperature=0 for deterministic single runs. Three GeneBench versions with different seeds show similar trends (§5.4.3, Figure 13) but no formal seed sensitivity analysis for model outputs is reported. Reasoning models that don't allow temperature changes are not analyzed for stochastic variation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The main Table 5 results are implied single-run at temperature=0 but not explicitly stated as such. The single-operator experiment states 'repeated the experiment three times' (§5.3). The number of runs for the main experiments is not explicitly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "GeneBench parameters (breed size 20%, 1-hour budget) are stated but no search over these hyperparameters is described. Fine-tuning uses QLoRA with early stopping but no hyperparameter search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The selection criterion is clearly defined: GeneBench selects the highest-RC individual from the Pareto front (Algorithm 1, Line 13, §3.3.1). For fine-tuning, loss-based early stopping is used. The breed size (20%) is justified by preventing population bloating (footnote 7)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs comparisons across 13 models and 5 benchmark-task combinations and reports multiple p-values (6e-18, 4e-8) without any mention of Bonferroni, Holm, or other family-wise error rate corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed GeneBench and evaluate it without acknowledging potential author-evaluation bias. No independent evaluation or discussion of this bias is present."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GeneBench transformations increase token counts 1.7x–6.2x (Table 6), meaning models process significantly more tokens, but this compute cost difference is not analyzed against the performance gains claimed. The 1-hour budget is mentioned but not varied to show performance as a function of compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark construct validity is a central theme of the paper. §2 systematically compares complexity metrics of existing benchmarks against SWE-Bench (Figure 1), demonstrating that existing benchmarks lack real-world complexity. §5.5 validates that GeneBench performance correlates with SWE-Bench performance."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "For the main experiments, LLMs are evaluated directly without scaffolding, eliminating the scaffold confound. For the SWE-Bench comparison (§5.5), all models use the same scaffold (Agentless), controlling for scaffold differences across model comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is a core motivation. §1 (L3) notes 'more than 90% of SWE-Bench problems are dated before the training cut-off date of most recent LLMs.' GeneBench is explicitly designed to address this by creating new transformed versions that did not exist during model training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "§5.4.1 avoids solution leakage in bug repair by selecting a different program's transformation as the ICL example rather than transformations of the same program. §1 (L2) discusses solution leakage in SWE-Bench where 'the issue report or the comments explicitly solving the issue' leak answers."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether original benchmark programs share structural similarities (e.g., similar coding patterns in HumanEval). The fine-tuning deduplication (Jaccard > 0.8) addresses near-duplicate training examples but not structural non-independence between train and test."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "NNMR (Nearest Neighbor Mismatch Rate, Table 6) measures the percentage of programs whose nearest neighbor in embedding space is NOT their transformed counterpart, serving as a concrete method to verify that transformed versions are sufficiently different from potentially-contaminated originals (19.4%–97.65% mismatch)."
       }
     }
   },
   "claims": [
     {
       "claim": "GeneBench increases relative complexity by 298% on average while readability decreases only 12%",
       "evidence": "Table 4 shows RC increases from avg 0.08 to 0.21 (ranging 75%–650% by benchmark), while RR decreases from 0.73 to 0.65. Average PyLint score improves from 9.75 to 9.87 (§5.2).",
       "supported": "strong"
     },
     {
       "claim": "LLMs suffer 14.9%–60.5% (avg 35.2%) performance drop across all tasks under GeneBench transformations",
       "evidence": "Table 5 reports Δ values across 13 models and 5 benchmark-task combinations, all negative, with p-value = 6e-18 (§5.3).",
       "supported": "strong"
     },
     {
       "claim": "Few-shot learning with GeneBench examples does not help LLMs bypass the complexity challenge, and often worsens performance",
       "evidence": "Table 5 Δ(IC) columns show average 41% drop under few-shot prompting. 'Except for a few cases, the performance drop is even higher than in the original experiment' (§5.4.1).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuned models still show 4.7%–30.3% (avg 17%) performance gap on GeneBench",
       "evidence": "Figure 11 shows CodeLlama-13-I and DeepSeekCoder-6.7-B fine-tuned on 106k GeneBench samples still underperform on all tasks compared to original benchmarks (§5.4.2).",
       "supported": "moderate"
     },
     {
       "claim": "LLMs' performance on GeneBench bug repair is similar to SWE-Bench, making GeneBench a proxy for real-world performance",
       "evidence": "Table 5: GeneBench repair rates (HumanEval Aft.=45.3%, ClassEval Aft.=20.6%) are closer to SWE-Sub (17.6%) than original benchmarks (66.4%, 32.1%). §5.5 provides detailed comparison.",
       "supported": "moderate"
     },
     {
       "claim": "Complexity, not representation shift, is the primary driver of LLMs' performance drop",
       "evidence": "§5.3.4: NNMR shows no correlation with average performance drop. CRUXEval-I has highest NNMR (97.65%) but lowest Δ. Figure 10 shows statistically significant complexity difference between Success_Failure and Success_Success groups (p=4e-8).",
       "supported": "strong"
     },
     {
       "claim": "Different versions of GeneBench produce notably different programs but consistently challenge LLMs",
       "evidence": "§5.4.3: Three versions show CodeBLEU 0.60–0.80, Jaccard 0.40–0.64 (Figure 12). Figure 13 shows all versions produce similar performance drop trends across models and tasks.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No code or data release",
       "detail": "Neither GeneBench's implementation nor its transformed benchmarks are publicly available. This is a significant limitation for a paper whose core contribution is a tool — readers cannot verify the transformations, reproduce results, or use GeneBench for their own evaluations."
     },
     {
       "flag": "No human evaluation of transformation naturalness",
       "detail": "The paper claims transformations maintain readability comparable to real-world code, supported only by automated metrics (PyLint score, readability metrics). No developer study validates whether the transformations look natural or representative. The Figure 2 vs Figure 3 comparison is anecdotal (one example)."
     },
     {
       "flag": "Limited real-world validation",
       "detail": "The SWE-Bench comparison (§5.5) validating GeneBench as a real-world proxy is limited to 94 instances in one task (bug repair). The paper acknowledges no real-world benchmarks exist for code reasoning or translation, leaving 3 of 4 tasks without real-world validation."
     },
     {
       "flag": "Fine-tuning limited to smallest models",
       "detail": "Fine-tuning experiments (§5.4.2) use only CodeLlama-13-I and DeepSeekCoder-6.7-B, the smallest models in their families. The 17% average gap may not hold for larger or proprietary models. Footnote 10 acknowledges this is due to GPU constraints, not experimental design choice."
     },
     {
       "flag": "No error bars on main results",
       "detail": "Table 5 reports single-point results at temperature=0 without any uncertainty quantification. While deterministic output eliminates stochastic variance, it doesn't account for sensitivity to other factors (prompt wording, which Pareto solution is selected, operator ordering)."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Primary real-world benchmark used for complexity comparison and validating GeneBench as a proxy for real-world performance."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark used as one of four GeneBench evaluation targets; foundational code LLM evaluation benchmark."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating llms on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Class-level code generation benchmark used as GeneBench evaluation target for program repair."
     },
     {
       "title": "CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution",
       "authors": ["Alex Gu", "Baptiste Rozière", "Hugh Leather", "Armando Solar-Lezama", "Gabriel Synnaeve", "Sida I Wang"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Code reasoning benchmark used for input/output prediction tasks in GeneBench evaluation."
     },
     {
       "title": "Top Leaderboard Ranking = Top Coding Proficiency, Always? EvoEval: Evolving Coding Benchmarks via LLM",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Related work on evolving benchmarks via LLM prompting; GeneBench differs by using genetic algorithms for code-to-code transformation."
     },
     {
       "title": "SWE-Bench+: Enhanced coding benchmark for llms",
       "authors": ["Reem Aleithan", "Haoran Xue", "Mohammad Mahdi Mohajer"],
       "year": 2024,
       "arxiv_id": "2410.06992",
       "relevance": "Identified solution leakage in SWE-Bench including SWE-Bench Verified, directly motivating GeneBench's contamination-resistant design."
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Repair tool used in the SWE-Bench comparison experiment to evaluate LLMs on real-world bug repair."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the evaluated model families (3 variants), showing significant performance drops under GeneBench transformations."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming–The Rise of Code Intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Evaluated model family (3 variants + R1); DeepSeekCoder-33-I showed lowest open-source performance drop, attributed to high code training ratio (87%)."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "One of two evaluated reasoning models showing the smallest performance drops under GeneBench, demonstrating reasoning abilities help with complexity."
     },
     {
       "title": "R2E: Turning any Github Repository into a Programming Agent Environment",
       "authors": ["Naman Jain", "Manish Shetty", "Tianjun Zhang", "King Han", "Koushik Sen", "Ion Stoica"],
       "year": 2024,
       "relevance": "Related real-world benchmark approach that collects methods from GitHub repos; used as comparison point for GeneBench's advantages."
     },
     {
       "title": "Concerned with Data Contamination? Assessing Countermeasures in Code Language Model",
       "authors": ["Jialun Cao", "Wuqi Zhang", "Shing-Chi Cheung"],
       "year": 2024,
       "arxiv_id": "2403.16898",
       "relevance": "Prior work on data contamination countermeasures for code LLMs; GeneBench operators are more complex than their semantic-preserving perturbations."
     },
     {
       "title": "Repository-level compositional code translation and validation",
       "authors": ["Ali Reza Ibrahimzada", "Kaiyao Ke", "Mrigank Pawagi"],
       "year": 2024,
       "arxiv_id": "2410.24117",
       "relevance": "Related work on repository-level code translation showing real-world translation is non-trivial, motivating GeneBench's complexity-aware evaluation."
     },
     {
       "title": "SemCoder: Training Code Language Models with Comprehensive Semantics",
       "authors": ["Yangruibo Ding", "Jinjun Peng", "Marcus J Min", "Gail Kaiser", "Junfeng Yang", "Baishakhi Ray"],
       "year": 2024,
       "arxiv_id": "2406.01006",
       "relevance": "Evaluated model instruction-tuned with execution data; showed less performance drop in some tasks, suggesting execution-aware training helps with complexity."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "GeneBench could be used by researchers to augment existing benchmarks, but the tool is not released, limiting immediate practical use."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Demonstrates a 35% average performance drop when adding real-world complexity to benchmarks, challenging the narrative that high benchmark scores indicate real-world capability."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the paper is about evaluation methodology."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The 'benchmarks are too easy' angle is mildly provocative but already well-known in the community."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released — readers cannot try GeneBench themselves."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From UIUC (well-known CS program); evaluates GPT-4o and o4-mini but the paper is not from a famous AI lab."
     }
   }
 }