ai-research-survey

scan-v4.json (33264B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "An evaluation of LLM code generation capabilities through graded exercises",
     "authors": [
       "Álvaro Barbero Jiménez"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2410.16292",
     "doi": "10.48550/arXiv.2410.16292"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about correlation with task difficulty, language popularity, publication date, and the 46.6%/37.4%/16% decomposition are all directly supported by results in Sections 4.1-4.5.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states '37.4% seems to be related to leakage' and uses 'attributed to' language. The surrogate model (Section 4.5) provides correlational evidence only — SHAP values on a 74.88%-accurate linear SVM do not establish causation. The study design is observational with no causal identification strategy.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only GPT-4o-mini on Codewars katas but generalizes broadly. The conclusions state 'current evaluations in the literature of the performance of state-of-the-art LLMs are, quite probably, overestimates of their real skill' — a claim about all LLMs based on one model on one platform.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 explicitly considers two competing hypotheses for the age effect: changed difficulty criteria vs. solution leakage. Section 4.5 addresses multicollinearity and acknowledges that features encode overlapping factors (e.g., easy katas have more completions AND more solutions available).",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Limitations) explicitly distinguishes between what was measured (solving programming exercises) and what broader claims this supports: 'actual work as a software developer involves not just solving programming challenges, but also writing documentation, finding and reporting bugs...' The gap between proxy and outcome is clearly acknowledged.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations' and contains substantive discussion of four specific limitations spanning multiple paragraphs.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: coverage limited to programming exercises (not full dev work), reproducibility constraints from Codewars ToS, absence of human-in-the-loop refinement, and surrogate model accuracy of only 74.88% introducing noise in the explanatory analysis.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states the evaluation 'only covers a small spectrum of this wide range of tasks' and enumerates what is excluded: documentation, bug finding, system design, requirements gathering, team collaboration, and roadmap definition.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The acknowledgments section thanks the Codewars community and IIC colleagues but mentions no grants, sponsors, or funding agencies.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation with Instituto de Ingeniería del Conocimiento and Universidad Autónoma de Madrid is clearly stated on the title page. The author evaluates a third-party product (OpenAI's GPT-4o-mini), not their own.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. The work appears to be unfunded academic research from a solo university-affiliated researcher.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'kata' and kyu difficulty levels (1–8) are explained with concrete examples, the Codewars scoring formula is stated explicitly (Equation 1), and 'code generation' is scoped to single-function synthesis from natural language descriptions.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The contribution is explicitly framed: evaluating GPT-4o-mini on 14,346 Codewars challenges across 8 languages and providing 'the first result that quantifies the impact of solutions leakage on the performance of an LLM for coding.'",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.4 provides a comprehensive review of code evaluation benchmarks (HumanEval, APPS, MBPP, ODEX, BigCodeBench, SWE-bench, etc.), showing how this work extends and differs from each prior effort.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "Section 5 (Limitations) explicitly states: 'we have decided to release neither the code of the developed botnet, nor the database of solutions proposed by the LLM' due to Codewars terms of use concerns.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The solutions database and evaluation results are explicitly withheld (Section 5). While Codewars katas are publicly available, the paper's collected dataset of LLM solutions and outcomes is not released.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions Python, Selenium, and the OpenAI API but provides no requirements.txt, library versions, or environment specification beyond naming the tools.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No reproduction instructions are provided. The code is explicitly withheld and no alternative reproduction path is described.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results are reported as point estimates (pass rates, Codewars scores). No confidence intervals or error bars appear in any figure or table.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper makes numerous comparative claims (LLM vs humans, across languages, across kyu levels) but no statistical significance tests are performed. Differences are stated from visual inspection of charts.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The SHAP analysis decomposes performance into difficulty (46.6%), leakage (37.4%), and language (16%) with full context. Per-language Codewars scores are reported (e.g., Python 36.1) against reference levels (87 for solving all kyu ≥4). Magnitudes are interpretable throughout.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The study uses ~14,346 katas but never justifies this sample size or discusses whether it is adequate for the claims made, particularly for rare kata-language combinations.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Each kata appears to be attempted once with a single LLM call. No variance across runs, seeds, or repeated attempts is reported.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Human performance from Codewars community statistics serves as a baseline throughout. Figure 7 directly compares LLM vs human success rates by difficulty level, and Figure 11 compares Codewars scores over time.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "The human baseline comes from current Codewars community statistics at the time of the study, making it contemporary and representative of human developer performance.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": false,
           "answer": false,
           "justification": "The system under evaluation is a single LLM (GPT-4o-mini) with a fixed prompt. There are no components to ablate.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses both raw pass rates (percentage of katas solved, Figures 7-8) and a weighted Codewars score metric (Equation 1, Figure 9) that emphasizes harder tasks. Timeout rates are also reported separately.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "Evaluation is entirely automated via Codewars unit tests (pass/fail). No human evaluation of generated code quality, readability, or correctness beyond test passing is performed.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Codewars uses a two-tier testing system: public unit tests visible to the solver, plus hidden tests used for final verification (Figure 1). The LLM only sees public tests; final evaluation uses hidden tests, functioning as a held-out set.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Extensive breakdowns by difficulty rank (8 levels, Figures 7-8), programming language (8 languages, Figures 8-9), and publication date (Figure 11) are provided.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.2 discusses COBOL's 'catastrophic results' with syntax errors, the complete failure on kyu 1-2 exercises, and timeout failures for JavaScript/Python on harder problems (yellow bars in Figure 8).",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports that the LLM fails completely on kyu 1-2 exercises (0% success), performs poorly on legacy languages, and that performance degrades significantly on newer challenges. These are central findings, not hidden.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Section 3 specifies 'GPT-4o-mini (version 2024-07-18)' — an exact version with snapshot date.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Figure 4 provides the full system prompt and user prompt with all text. The placeholders (%LANGUAGE%, %DESCRIPTION%, %SOLUTION_TEMPLATE%, %UNIT_TESTS%) are filled from publicly available Codewars kata content, making the prompts fully reconstructable.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "No LLM inference hyperparameters are reported — temperature, top-p, max tokens, and other API parameters that affect output quality are not mentioned anywhere.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The LLM receives a single prompt and produces a single response. The Selenium bot infrastructure is evaluation tooling, not LLM scaffolding.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3 documents the data collection from Codewars, the 8 language selection rationale, that 57 katas (0.4%) were discarded due to regex/Selenium incompatibilities, and the resulting distribution across languages and difficulty levels (Figure 5).",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Raw data (kata solutions, pass/fail results) is explicitly withheld per Section 5: 'we have decided to release neither the code of the developed botnet, nor the database of solutions proposed by the LLM.'",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3 and Figure 3 describe the Selenium bot system, interaction with the Codewars API and OpenAI API, the account creation/deletion procedure, and responsible crawling practices.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The data source is Codewars, a public coding challenge platform. The kata selection is the full set available for 8 languages.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Figure 3 diagrams the full pipeline: kata downloader bot → solution generator bot (via OpenAI API) → kata attempter bot → result verification. Exclusion criteria (57 regex-incompatible katas) are documented with counts.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The model version is specified (GPT-4o-mini 2024-07-18) but the training data cutoff date is not stated. Without this, temporal contamination cannot be precisely assessed.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "This is a central theme. Section 4.4 discusses how ~38,500 public GitHub repos contain Codewars solutions, and the SHAP analysis (Section 4.5) attributes 37.4% of performance to leakage-related features. The paper also cites similar concerns with APPS and SWE-bench.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination is the paper's core contribution. Section 4.4 discusses solution availability in public repos, Section 4.5 quantifies contamination impact via surrogate model analysis, and Section 6 proposes temporal evaluation as mitigation.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study. The evaluation is fully automated LLM-on-benchmark.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The study evaluates an LLM on coding challenges.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Human performance data comes from aggregate Codewars community statistics.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "The paper notes GPT-4o-mini is 'approximately 30 times cheaper' than GPT-4o (Section 3) but never reports the actual API cost for the ~14,346 kata attempts.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget is reported — neither total API spend, wall-clock time for the full evaluation, nor hardware used for the bot infrastructure.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "Each kata appears to be attempted a single time. No multiple seeds or repeated sampling is performed, despite LLM outputs being stochastic.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper never explicitly states how many attempts per kata were made. It appears to be a single attempt but this is not stated.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "For the LLM evaluation, API parameters (temperature, sampling) are not reported, let alone searched. The surrogate model hyperparameters are reported (Table 1) but the main LLM evaluation hyperparameters are absent.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": true,
           "justification": "For the surrogate model, 5-fold cross-validation is used to select the best configuration (Section 4.5, Table 1). The linear SVM is chosen over XGBoost because it produces equivalent accuracy (74.88% vs 74.91%) with greater interpretability.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": false,
           "answer": false,
           "justification": "No statistical significance tests are performed in the paper, so correction for multiple comparisons is structurally inapplicable.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors build the evaluation framework and surrogate model but do not discuss how their design choices (feature selection, model choice, clustering of SHAP features into difficulty/leakage/language groups) could bias the results.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": false,
           "justification": "The paper tests only GPT-4o-mini, choosing it over GPT-4o for cost (30x cheaper). No performance-compute comparison is made, despite this being directly relevant to interpreting the results.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": true,
           "justification": "This is a central theme. The paper extensively discusses whether coding benchmarks (Codewars, HumanEval, APPS, SWE-bench) actually measure LLM coding capability vs. memorization. Section 1.4 reviews construct validity issues and Section 6 argues current evaluations overestimate real skill.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No scaffolding is involved. The LLM receives a single prompt and produces a single response; no agentic workflow or tool use.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.4 analyzes how kata publication date correlates with LLM success, finding older katas with more time for solutions to spread are easier for the LLM. The SHAP analysis includes 'days since publication' as a leakage-related feature.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether the evaluation setup (public unit tests provided in the prompt) leaks information about solutions that wouldn't be available in a real coding scenario.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether Codewars katas share structural similarities, whether solutions to similar katas in training data could transfer, or whether kata families create non-independence in the evaluation.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": true,
           "justification": "The surrogate model with SHAP analysis (Section 4.5) provides a concrete, quantitative method for estimating leakage impact. Features encoding leakage likelihood (completions, publication date) are systematically analyzed via non-interventional SHAP with 100K sampling points.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4o-mini outperforms humans on easy katas (rank 8–7) but fails completely on rank 1–2 katas.",
       "evidence": "Figure 7 shows LLM success rate exceeds human completion rate for ranks 8–7, while rank 1–2 shows zero LLM successes versus non-zero human completion across the entire evaluation corpus.",
       "supported": "strong"
     },
     {
       "claim": "~37.4% of LLM coding performance on Codewars is attributable to solution leakage into training data.",
       "evidence": "SHAP analysis on a linear SVM surrogate model groups features by hypothesized cause; leakage-proxy features (days since publication, total completions) account for 37.4% of total absolute SHAP value. The model itself has only 74.88% accuracy and the attribution is indirect.",
       "supported": "weak"
     },
     {
       "claim": "LLM performance varies significantly by programming language, with legacy languages (COBOL, Fortran) showing dramatically degraded results.",
       "evidence": "Figure 8 shows near-zero completion rates for COBOL across all difficulty levels with syntax errors noted on visual inspection; Figure 9 shows large Codewars score gaps correlated with GitHub code volume (Figure 10).",
       "supported": "strong"
     },
     {
       "claim": "Newer Codewars katas are harder for the LLM than for humans, consistent with solution leakage benefiting older challenges disproportionately.",
       "evidence": "Figure 11 shows LLM performance declining more steeply with kata recency than human performance, with the LLM falling below human levels on basic/intermediate tasks from recent years. Two alternative explanations are noted but not distinguished.",
       "supported": "moderate"
     },
     {
       "claim": "Current LLM code generation benchmark evaluations likely overestimate actual model skill due to data contamination.",
       "evidence": "The surrogate model attributes ~37.4% of performance to leakage proxies, and 38,500 public GitHub repos contain Codewars solutions. However, this rests on indirect measurement from a single model with no controlled intervention.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "GPT-4o-mini was evaluated on 14,346 Codewars coding challenges across 8 programming languages using a bot network that submits solutions against the platform's hidden test sets. The model outperforms humans on easy exercises but completely fails on the hardest challenges (rank 1–2), and performance degrades catastrophically for low-resource and legacy languages (especially COBOL). A surrogate model with SHAP analysis estimates that ~46.6% of performance is explained by task difficulty, ~37.4% by solution leakage into training data (proxied by kata age and user completion counts), and ~16% by programming language popularity — suggesting current code generation benchmarks substantially overestimate true LLM capability.",
   "red_flags": [
     {
       "flag": "Single model, broad conclusions",
       "detail": "Only GPT-4o-mini is evaluated, but conclusions generalize to 'current evaluations of state-of-the-art LLMs' being 'quite probably overestimates.' One model on one platform cannot support this claim."
     },
     {
       "flag": "Code and data withheld — irreproducible by design",
       "detail": "The paper explicitly states code and evaluation data are not released due to Codewars TOS concerns. Results cannot be independently verified or reproduced by any third party."
     },
     {
       "flag": "Causal attribution via proxy features in surrogate model",
       "detail": "The 37.4% leakage figure is derived from grouping correlated features in an SVM surrogate model by hypothesized cause, not from any direct measurement of training data overlap or controlled experiment. The surrogate model itself achieves only 74.88% accuracy."
     },
     {
       "flag": "No statistical tests or uncertainty quantification",
       "detail": "All comparative claims (LLM vs. humans, between languages, age trends) are supported only by visual inspection of bar charts and trend lines with no significance tests, confidence intervals, or variance estimates."
     },
     {
       "flag": "Training cutoff not disclosed",
       "detail": "Despite the central focus on training data contamination, the training cutoff date for GPT-4o-mini is never stated, making it impossible to reason precisely about which katas were available during training."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval / Codex)",
       "relevance": "Foundational benchmark paper for LLM code generation; introduces pass@k metric and discusses contamination from public repos; directly compared throughout"
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "relevance": "Prior benchmark evaluating LLMs on programming challenges scraped from competitive platforms; directly compared methodology and results including difficulty scaling"
     },
     {
       "title": "Program Synthesis with Large Language Models (MBPP)",
       "relevance": "Prior LLM code evaluation showing similar difficulty scaling and multi-turn improvement; compared single-turn vs. multi-turn performance"
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Most realistic code benchmark; cited as state-of-the-art comparison and as another benchmark with known contamination from public GitHub repos"
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "relevance": "Contemporary benchmark showing 60% ceiling even for best models; compared scope and findings on library-usage difficulty"
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "relevance": "Used to justify model selection (GPT-4o-mini ranked #2 for coding) and as an example of crowd-sourced evaluation methodology"
     },
     {
       "title": "Execution-Based Evaluation for Open-Domain Code Generation (ODEX)",
       "relevance": "Prior benchmark showing library popularity correlates with LLM performance; directly corroborates the language-popularity finding"
     },
     {
       "title": "Open LLM Leaderboard v2",
       "relevance": "Example of contamination forcing benchmark replacement in the general LLM evaluation space; motivates the paper's code-specific contamination concern"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly actionable for practitioners who rely on benchmark scores to select LLMs for coding tasks — quantifies how much contamination may inflate reported capability."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Provides a novel quantification of leakage (~37.4%) that challenges trust in existing code benchmarks, though contamination as a concern is broadly acknowledged."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or harm concerns raised; purely an evaluation methodology paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implies state-of-the-art benchmark results are inflated, which is mildly provocative, but the framing is measured and academic."
     },
     "demo_ability": {
       "score": 1,
       "justification": "The exact study cannot be replicated (code withheld, TOS issues), but readers can manually test LLMs on Codewars challenges to spot-check the finding."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "GPT-4o-mini (OpenAI) is a recognizable product, but the paper is from a small Spanish research institute with no major lab affiliation."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "39274918",
         "title": "Better Call GPT: Comparing large language models against lawyers [pdf]",
         "points": 389,
         "comments": 264,
         "url": "https://news.ycombinator.com/item?id=39274918",
         "created_at": "2024-02-06T15:04:39Z"
       },
       {
         "hn_id": "42021222",
         "title": "Fast and Accurate Deep Reconfigurable Spiking Inference Accelerator Architecture",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42021222",
         "created_at": "2024-11-01T20:28:32Z"
       },
       {
         "hn_id": "41926182",
         "title": "We discovered a way to measure LLM bias while building a recruitment tool",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=41926182",
         "created_at": "2024-10-23T15:41:33Z"
       },
       {
         "hn_id": "42576715",
         "title": "Reinforcement Learning for Multi-Intersection Traffic Signal Control",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42576715",
         "created_at": "2025-01-02T17:51:07Z"
       },
       {
         "hn_id": "38177348",
         "title": "CleanCoNLL: A Nearly Noise-Free Named Entity Recognition Dataset",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38177348",
         "created_at": "2023-11-07T14:47:31Z"
       }
     ],
     "top_points": 389,
     "total_points": 394,
     "total_comments": 265
   }
 }