ai-research-survey

scan.json (31392B)

---

 {
   "paper": {
     "title": "JavaBench: A Benchmark of Object-Oriented Code Generation for Evaluating Large Language Models",
     "authors": [
       "Jialun Cao",
       "Zhiyong Chen",
       "Jiarong Wu",
       "Shing-Chi Cheung",
       "Chang Xu"
     ],
     "year": 2024,
     "venue": "International Conference on Automated Software Engineering",
     "arxiv_id": "2406.12902",
     "doi": "10.1145/3691620.3695470"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is publicly released at https://github.com/java-bench/JavaBench as stated in the abstract and Section 8: 'We released the implementation and all associated publicly available data.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (projects, test suites, skeletons) is released via the same GitHub repository. Section 8 confirms 'all associated publicly available data' are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware ('two NVIDIA RTX 6000 Ada GPUs, each with 48GB') and specific model IDs, but provides no requirements.txt, Dockerfile, or detailed software dependency versions needed to recreate the evaluation environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a GitHub link and describes the methodology, but does not include step-by-step reproduction instructions, scripts to replicate experiments, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 5 and 6 are reported as point estimates (e.g., '72.33% Compilation@1') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims ('holistic synthesis was generally better,' 'selected context yield the best overall results') based solely on comparing raw numbers without any significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context throughout, e.g., 'WizardCoder dropped 51.43% (80.00% - 28.57%)' and '23.62% (= 34.34%-10.72%) drop.' Raw values and differences are consistently provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of n=5 generated solutions is acknowledged only as 'Considering the cost and time, we set n to 5 and k to 1 and 5, following the previous study [28].' No power analysis or adequacy justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite generating 5 samples per configuration, no standard deviations or variance measures are reported across runs. Pass@k is computed as an expectation using the unbiased estimator, but no spread measures are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five LLMs are compared against each other (Table 5) and against a human baseline of 282 undergraduate students (90.93% average, Table 2). The paper also compares synthesis strategies and context settings."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The studied LLMs (WizardCoder-15B, DeepSeek-6.7b/33b, Phind-34B, gpt-3.5-turbo) were released in 2022-2023. For a 2024 ASE paper, notable omissions include GPT-4, Claude, and other frontier models. The 34B parameter cap due to hardware limitations is acknowledged but not mitigated."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 ablates context settings (maximum/minimum/selected) and RQ1/RQ3 compare synthesis strategies (holistic/independent/incremental with sequential/reverse/random orders), measuring the impact of each factor."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three progressive metrics are used: Completion@k, Compilation@k, and Pass@k, each at two granularities (class-wise and test-wise), as defined in Section 3.2.2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The 282 students validated the benchmark by completing the projects (Section 2.2.3), not by evaluating LLM outputs. The bad case analysis in RQ4 involves manual inspection of failures but is not systematic human evaluation of generated code."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is used as a fixed test set. Models are evaluated without any fine-tuning or selection decisions on the benchmark data. The benchmark was kept confidential and not used in model training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by project (P1-P4), by model, by synthesis strategy, by context setting, and by evaluation granularity (class-wise vs. test-wise) in Tables 5 and 6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 4.4) provides detailed bad case analysis with five specific failure examples (Listings 3-7): inheritance errors, encapsulation violations, illegal inheritance, documentation non-following, and trivial implementations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that no project can be correctly completed at project-wise granularity (all-zero Pass@k under all settings). It also reports reversed order in incremental synthesis does not help (Finding 7)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'no project can be correctly completed' (confirmed by all-zero project-wise Pass@k), LLMs far behind students (48.24% best avg test-wise Pass@5 vs 90.93% human), method signature as optimal context (Table 6 confirms selected context best). The '41.17% Pass@5' in the abstract aligns approximately with Section 4.1's '41.7%' for a specific setting."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'holistic synthesis yielded better performance' and 'providing too much or too little context has a negative impact' are supported by controlled single-variable manipulation: varying synthesis strategy while fixing context (RQ1), and varying context while fixing strategy (RQ2)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'LLMs are far behind undergraduate students' broadly, and findings like 'LLMs' capability to handle OOP features is far behind that of undergraduates' generalize beyond the 5 tested models and 4 student game projects. The threats section notes model generalization but the claims remain broad."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses prompt engineering as an alternative explanation ('a well-designed prompt could yield better performance'). Section 4.1 discusses that differences from ClassEval could be due to 'different programming languages (Python VS. Java) and code granularities (Class-level VS. Project-level),' offering specific alternative explanations for observed differences."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Pass@k on 4 entry-level course game projects and frames this as measuring 'LLMs' capability to handle OOP features' and 'project-level Java programming ability.' The gap between pass rates on 4 specific student assignments and general Java OOP capability is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 specifies exact model identifiers: 'WizardCoder-15B-V1.0', 'deepseek-coder-6.7b-instruct', 'deepseek-coder-33b-instruct', 'Phind-CodeLlama-34B-v2', 'gpt-3.5-turbo-1106' with sizes and release dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Listing 2 provides the full prompt template with system message and instruction text. The placeholder fill values (context and class code) come from the released benchmark and are thoroughly explained with examples in Section 3.1.1 and Figure 2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states 'nucleus sampling... with a temperature of 0.2' and 'five solution samples were randomly generated.' Context truncation to 8192 characters is also specified (Section 3.1.1)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation involves direct prompting of LLMs to complete TODO methods, with no tools, feedback loops, or agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents benchmark construction (project selection, canonical solutions, test construction). Section 3.1.1 describes context extraction using jdeps for dependency analysis. The truncation to 8192 characters is specified with statistics (53.3% contexts truncated)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Threats to Validity' provides a dedicated discussion of limitations including benchmark construction, model generalizability, prompt engineering, and data contamination."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: 'we only studied five LLMs due to time and hardware limits,' 'the projects in JavaBench were kept confidential,' and quality of natural language descriptions could affect results. These go beyond generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes 'the conclusion may not be able to generalize to other LLMs' but does not explicitly state what the results do NOT show, such as limitations to entry-level game projects, or inapplicability to industrial Java development. The boundaries are stated generically."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states 'We released the implementation and all associated publicly available data at https://github.com/java-bench/JavaBench.' The benchmark projects, test suites, and canonical solutions are available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes benchmark construction in detail: projects from an entry-level Java course, canonical solutions by experienced programmers with 5+ years of experience and cross-validation, test suites manually constructed with coverage metrics reported."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The 282 students are described as 'undergraduate students' in an 'entry-level Java course' across '2019 to 2022,' but no details about how the course was selected, whether these students are representative, or potential selection biases are provided."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: project design (Section 2.2.1) → canonical solution construction (cross-validated) → test suite construction with coverage metrics (Section 2.2.2) → student validation (Section 2.2.3) → LLM evaluation pipeline (Section 3). Student exclusion criteria are documented in footnote 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the provided paper text. The authors are from HKUST and Nanjing University but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: HKUST Department of Computer Science and Engineering, Guangzhou HKUST Fok Ying Tung Research Institute, and State Key Laboratory for Novel Software Technology at Nanjing University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors are from academic institutions and evaluate third-party models, suggesting low conflict risk, but the absence of a funding disclosure statement itself is the issue."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 lists model release dates (e.g., June 2023, Nov 2022) but does not state training data cutoff dates. Release date is not equivalent to training cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly discusses contamination: 'LLMs having seen the canonical code during training could lead to exaggerated scores, known as data contamination. However, the projects in JavaBench were kept confidential.' Section 2.2.1 notes students must keep assignments confidential."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination by arguing the benchmark projects were kept confidential (Section 2.2.1: 'Students are required to keep the course assignments and canonical solutions confidential for academic integrity, which reduces the data contamination threat')."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. The use of student data from a Java course was not pre-registered as a research study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite using data from 282 undergraduate students."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Students are described only as 'undergraduate students' in an 'entry-level Java course' from 2019-2022. No information on gender, age, geographic distribution, or prior programming experience is reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 states: 'we omit course withdrawals, non-submissions, and blank project submissions from the count because these cases do not attempt to complete the project.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is not an experimental study with conditions. Students completed course assignments as part of their normal coursework; no randomization to conditions occurred."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — students completed standard course assignments. There were no experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 explains that withdrawals, non-submissions, and blank submissions were excluded. Per-project participant counts are reported (62-79 students per project, 282 total)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or per-example costs are reported. The paper mentions hardware but not the cost of running experiments across 5 models × multiple configurations × 5 samples."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two NVIDIA RTX 6000 Ada GPUs, each with 48GB' but does not report total GPU hours, wall-clock time, or total compute budget for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper generates 5 samples per configuration using nucleus sampling but does not report sensitivity to random seeds or variation across different sampling runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states 'five solution samples were randomly generated with a temperature of 0.2.' The number of samples (n=5) and k values (1 and 5) are explicitly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.2 and context window 8192 tokens are used following prior work without reporting any search budget. No justification is provided for why these values were chosen beyond citing prior studies."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results for all tested configurations (3 context settings × 3-5 synthesis strategies × 5 models) in Tables 5 and 6, not just the best. The systematic exploration across configurations avoids cherry-picking."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so there are no tests to correct for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate models on their own benchmark without acknowledging potential bias from benchmark design choices that might favor or disfavor certain models."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 6.7B to 34B parameters (plus GPT-3.5), but performance is not analyzed as a function of compute. The larger DeepSeek-33b does not always outperform smaller models, but this relationship is not explicitly analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper validates the benchmark through human performance (90.93% average from 282 students), test coverage metrics (92% class, 87% function, 86.75% line coverage), code complexity metrics, and systematic comparison with 24 existing benchmarks in Table 1."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models are evaluated via direct prompting with the same prompt template and context settings."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper argues the benchmark was kept confidential through academic integrity requirements (Section 2.2.1), reducing the risk that models trained on internet data could have seen the benchmark solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (context settings, project descriptions, class skeletons) provides hints that would not be available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 4 projects (all game-themed, from the same course) share structural similarities that could affect the independence of performance measurements across projects."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. The paper relies solely on the confidentiality argument without using canary strings, membership inference, or n-gram overlap analysis."
       }
     }
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "claims": [
     {
       "claim": "LLMs are far behind undergraduate students in project-level Java programming: no project can be correctly completed by any studied LLM, and the best average test-wise Pass@5 is 48.24% vs. 90.93% human average.",
       "evidence": "Table 5 shows all-zero project-wise Pass@k under all settings. Section 4.1 reports 48.24% average test-wise Pass@5 for holistic strategy. Table 2 reports 90.93% average human score across 282 students.",
       "supported": "strong"
     },
     {
       "claim": "Holistic synthesis yields better performance than independent and incremental synthesis strategies across all LLMs.",
       "evidence": "Table 5 shows holistic averages of 91.73% Completion@1, 72.33% Compilation@1, 70.92% Pass@1 (class-wise) vs. independent 79.70%/62.89%/61.76% and incremental 75.84%/60.81%/59.76%. Finding 2, Section 4.1.",
       "supported": "moderate"
     },
     {
       "claim": "Selected context (method signatures only) strikes the optimal balance, outperforming both maximum and minimum context settings.",
       "evidence": "Table 6 shows selected context achieves 70.92% class-wise Pass@1 vs. 64.56% (maximum) and 37.47% (minimum). Test-wise: 27.40% vs. 26.55% and 3.79%. Finding 5-6, Section 4.2.",
       "supported": "moderate"
     },
     {
       "claim": "Random and sequential order in incremental synthesis can yield up to 6% improvement over reversed order.",
       "evidence": "Figure 5 shows for DeepSeek-Coder-6.7b at Pass@5, random order yields 86.07% vs. 80% for reversed. Finding 7, Section 4.3.",
       "supported": "weak"
     },
     {
       "claim": "Finer-grained evaluation granularities (class-wise and test-wise) are necessary to capture nuanced performance differences that would be all-zero at project-wise granularity.",
       "evidence": "Section 4.1 shows class-wise scores around 65% while project-wise Pass@k is all zeros. The gap between class-wise and test-wise is up to 49.92%. Finding 3.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "JavaBench introduces a project-level Java benchmark with 4 OOP projects (389 methods, 106 classes, 92% test coverage) validated by 282 students averaging 90.93%. All five studied LLMs fail to complete any project correctly at project-wise granularity, with the best test-wise Pass@5 averaging 48.24% under holistic synthesis with selected context. Method signatures as context outperform both maximum and minimum context settings, and holistic synthesis outperforms independent and incremental strategies across all models.",
   "red_flags": [
     {
       "flag": "Very small benchmark scope",
       "detail": "Only 4 projects, all entry-level game-themed course assignments from the same institution. This narrow domain may not represent real-world industrial Java development or diverse OOP patterns."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims across strategies, contexts, and models are based on raw numerical differences without any significance testing. With n=5 stochastic samples, observed differences could be due to sampling noise."
     },
     {
       "flag": "Outdated model selection",
       "detail": "The 5 studied models (max 34B parameters, all from 2022-2023) exclude frontier models like GPT-4, Claude, or larger open-source models. Conclusions about 'LLMs' broadly are drawn from a limited and non-frontier model set."
     },
     {
       "flag": "Unfair human-LLM comparison",
       "detail": "Students had weeks to complete assignments with access to course materials and could debug iteratively. LLMs had single-pass generation with fixed prompts. The comparison does not control for time, resources, or iterative refinement opportunities."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the dominant code generation benchmark that JavaBench seeks to complement with project-level Java evaluation."
     },
     {
       "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation",
       "authors": ["Xueying Du", "Mingwei Liu"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Class-level code generation benchmark that JavaBench extends to project-level; synthesis strategies and evaluation design are adapted from this work."
     },
     {
       "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
       "authors": ["Jia Li", "Ge Li"],
       "year": 2024,
       "arxiv_id": "2405.19856",
       "relevance": "Project-level Python benchmark that JavaBench positions itself against as the first project-level Java benchmark."
     },
     {
       "title": "OOP: Object-Oriented Programming Evaluation Benchmark for Large Language Models",
       "authors": ["Shuai Wang", "Liang Ding"],
       "year": 2024,
       "arxiv_id": "2401.06628",
       "relevance": "Only prior benchmark claiming OOP evaluation, but JavaBench argues it lacks actual code context and only mentions OOP concepts in prompts."
     },
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models",
       "authors": ["Hao Yu", "Bo Shen"],
       "year": 2023,
       "arxiv_id": "2302.00288",
       "relevance": "Function-level benchmark covering Python and Java code generation that JavaBench extends to project-level scope."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark, one of the most widely used Python code generation benchmarks."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "authors": ["Fengji Zhang", "Bei Chen"],
       "year": 2023,
       "doi": "10.18653/v1/2023.emnlp-main.151",
       "relevance": "Repository-level code completion technique using retrieval augmentation, relevant to JavaBench's context selection methodology."
     },
     {
       "title": "Concerned with Data Contamination? Assessing Countermeasures in Code Language Model",
       "authors": ["Jialun Cao", "Wuqi Zhang"],
       "year": 2024,
       "arxiv_id": "2403.16898",
       "relevance": "Studies data contamination in code LLMs, directly relevant to benchmark evaluation validity."
     },
     {
       "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation",
       "authors": ["Max Schäfer", "Sarah Nadi"],
       "year": 2024,
       "doi": "10.1109/TSE.2023.3334955",
       "relevance": "Empirical study of LLM-based test generation, relevant to the intersection of code generation evaluation and test quality."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F Liu", "Kevin Lin"],
       "year": 2024,
       "relevance": "Explains LLMs ignoring mid-context information, cited to explain completion errors where models skip TODO markers in long contexts."
     },
     {
       "title": "Time Travel in LLMs: Tracing Data Contamination in Large Language Models",
       "authors": ["Shahriar Golchin", "Mihai Surdeanu"],
       "year": 2023,
       "arxiv_id": "2308.08493",
       "relevance": "Framework for detecting data contamination in LLMs, directly relevant to benchmark validity concerns."
     },
     {
       "title": "CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X",
       "authors": ["Qinkai Zheng", "Xiao Xia"],
       "year": 2023,
       "doi": "10.1145/3580305.3599790",
       "relevance": "Introduces HumanEval-X multilingual benchmark, one of the few existing benchmarks including Java."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released benchmark with leaderboard that Java/OOP-focused teams could use to evaluate LLMs, but the 4-project scope limits practical adoption."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "LLMs struggling with project-level code is expected; the gap from student performance (48% vs 91%) quantifies it but doesn't challenge conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; this is a capability evaluation benchmark."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild drama in 'LLMs are far behind undergraduates' framing, but the result is unsurprising for project-level tasks."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Benchmark released on GitHub with a public leaderboard at java-bench.github.io, allowing model developers to test their systems."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates ChatGPT (gpt-3.5-turbo) and DeepSeek but comes from academic labs, not a well-known AI company."
     }
   }
 }