ai-research-survey

scan-v5.json (21410B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "JavaBench: A Benchmark of Object-Oriented Code Generation for Evaluating Large Language Models",
     "authors": [
       "Jialun Cao",
       "Zhiyong Chen",
       "Jiarong Wu",
       "S. Cheung",
       "Chang Xu"
     ],
     "year": 2024,
     "venue": "International Conference on Automated Software Engineering",
     "arxiv_id": "2406.12902",
     "doi": "10.1145/3691620.3695470"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'at most 41.17% Pass@5 in a more relaxed evaluation' but Section 4.1 Finding 4 reports 'The best average test-wise Pass@5 in JavaBench is 48.24%' — a significant numerical inconsistency suggesting the abstract was not updated with final results.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Comparative claims (holistic synthesis outperforms independent/incremental; selected context outperforms maximum/minimum) are supported by controlled ablation experiments in RQ1-RQ3 across five LLMs with consistent results.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Findings are presented broadly ('LLMs are far behind undergraduate students,' 'LLMs' capability to handle OOP features') based on only 5 models with no GPT-4 or frontier model coverage; the main findings text does not consistently bound claims to the studied models.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The student comparison is not controlled for testing conditions (students work iteratively with full documentation, LLMs make a fixed number of attempts); no alternative explanations for the LLM-student gap are considered.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes Completion@k (syntactic completion), Compilation@k (compilability), and Pass@k (semantic correctness via test suite), and characterizes test coverage at 87-92%, clearly separating what is measured from broader code quality.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Threats to Validity' is a dedicated section discussing benchmark construction quality, LLM generalizability, prompt engineering variance, and data contamination.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: only 5 LLMs studied due to 'time and hardware limits,' documentation quality affecting generation, prompt engineering variance, and confidentiality as contamination mitigation — concrete enough to be more than boilerplate.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The Threats section acknowledges the 5-LLM limitation but the main findings text still generalizes to 'LLMs' broadly without consistently stating which conclusions should not be extrapolated beyond the studied setting.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no acknowledgments section, funding statement, or grant disclosure anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (HKUST and Nanjing University) are clearly listed on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, financial interests declaration, or conflict-of-interest disclosure appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "OOP features (encapsulation, inheritance, polymorphism), context settings (maximum/minimum/selected), synthesis strategies (holistic/independent/incremental), and evaluation metrics (Completion/Compilation/Pass@k) are all explicitly defined.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The Contributions section explicitly lists three contributions: significance (first project-level Java OOP benchmark), novelty (systematic evaluation design), and evaluation (extensive experiments with findings).",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 systematically compares 24 existing benchmarks across language, granularity, and scale; Section 6 discusses how JavaBench extends, differs from, and complements ClassEval, DevEval, RepoEval, and OOPEval.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that OOP features (inheritance, polymorphism) inherently require multi-class project contexts that function-level benchmarks cannot assess, making project-level Java a necessary condition for the claimed measurement.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 reports cyclomatic and cognitive complexity per project; Table 2 shows human performance mean and standard deviation per project; however, there are only 4 projects with similar difficulty (90-95% human pass rate), providing limited range.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly shows no ceiling effect (best LLM 48.24% test-wise Pass@5 vs 90.93% human) and notes project-wise evaluation yields all-zero results, indicating the benchmark is appropriately challenging without a floor problem at coarser granularity.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": true,
           "justification": "282 undergraduate students completed the four projects over four academic years, achieving a 90.93/100 average pass rate; per-project breakdown with min/max/mean/std is provided in Table 2.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly argues execution-based Pass@k over similarity metrics (BLEU), defines the Pass@k formula, and justifies the class-wise and test-wise granularities as capturing nuanced partial success shadowed by project-wise all-zero evaluation.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "Contamination resistance relies entirely on academic confidentiality of student assignments — no canary strings, temporal splits, or dynamic generation; the paper acknowledges this as a threat but offers no technical countermeasure.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss how the benchmark will remain useful as LLMs improve, does not project a saturation timeline, and does not provide an update or versioning plan.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.4 analyzes five categories of benchmark failure modes in generated code: completion errors, inheritance errors, encapsulation errors, illegal inheritance, documentation non-following, and trivial implementations — with concrete code examples.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full implementation is publicly released at https://github.com/java-bench/JavaBench with a leaderboard at https://java-bench.github.io/leaderboard.html enabling reproduction of reported numbers.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Section 2 documents benchmark format, project descriptions, exercised Java concepts, test construction methodology, code coverage metrics, and human performance; Table 3 provides complete code and test statistics per project.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The GitHub URL is provided and access is clearly open, but no license is stated in the paper, leaving the terms of reuse legally ambiguous.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes what the benchmark measures (project-level Java OOP code generation) but does not specify what should NOT be concluded from benchmark results or warn against misuse cases.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs are far behind undergraduates on project-level Java OOP tasks: the best LLM achieves at most 48.24% test-wise Pass@5 vs 90.93% for undergraduates, and no LLM correctly completes any project in project-wise evaluation.",
       "evidence": "Table 5 and Section 4.1 Finding 4 report all-zero project-wise Pass@5 across all 5 LLMs; 48.24% best average test-wise Pass@5 vs 90.93% student average in Table 2.",
       "supported": "strong"
     },
     {
       "claim": "95.8% of existing code generation benchmarks target Python, and only 5 involve Java, all at function level.",
       "evidence": "Table 1 surveys 24 benchmarks; 23/24 involve Python; only Concode, HumanEval-X, MBXP, MultiPL-MBPP, CoderEval involve Java at function level.",
       "supported": "strong"
     },
     {
       "claim": "Holistic synthesis (generating all methods in a class in one pass) consistently outperforms independent and incremental synthesis strategies.",
       "evidence": "Table 5 shows holistic achieves best Completion@1 (91.73%), Compilation@1 (72.33%), and Pass@1 (70.92%) averaged across all LLMs; consistent across all 5 models.",
       "supported": "strong"
     },
     {
       "claim": "Selected context (providing only method signatures of dependent classes) strikes the optimal balance, outperforming both maximum and minimum context settings.",
       "evidence": "Table 6 shows selected context achieves 70.92% class-wise Pass@1 vs 64.56% maximum and 37.47% minimum; minimum context produces near-zero test-wise pass rates.",
       "supported": "strong"
     },
     {
       "claim": "Providing too much context (maximum) or too little context (minimum) both degrade project-level code generation performance.",
       "evidence": "Table 6 shows minimum context yields near-zero test-wise Pass@1 across all models; maximum context improves some models but degrades others (e.g., DeepSeek-33b drops 23.62pp on P4).",
       "supported": "moderate"
     },
     {
       "claim": "AssertionFailedError and IllegalArgumentException account for 76.63% of test-failing errors in LLM-generated code.",
       "evidence": "Figure 6 exception distribution analysis reports AssertionFailedError at 50.75% and IllegalArgumentException at 25.88% of test failures.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "JavaBench demonstrates a substantial gap between LLM and human performance on project-level Java OOP code generation: the best LLM (DeepSeek-33b) achieves 48.24% test-wise Pass@5 versus 90.93% for undergraduates, with all LLMs scoring 0% on project-wise evaluation. Holistic synthesis (generating all methods in a class simultaneously) outperforms independent and incremental strategies across all five evaluated LLMs. The benchmark identifies three main error types in LLM-generated Java: completion failures, OOP-specific compilation errors (inheritance, encapsulation, polymorphism violations), and test failures from documentation non-following and trivial implementations. Selected context (method signatures of dependent types only) provides the optimal balance between information richness and input token efficiency.",
   "red_flags": [
     {
       "flag": "Abstract-body number discrepancy",
       "detail": "The abstract states 'at most 41.17% Pass@5 in a more relaxed evaluation' but Section 4.1 Finding 4 reports 'The best average test-wise Pass@5 in JavaBench is 48.24%'. These numbers are inconsistent, suggesting the abstract was not updated with final experimental results."
     },
     {
       "flag": "Extremely small benchmark scale",
       "detail": "JavaBench contains only 4 projects, making it difficult to draw statistically robust conclusions about LLM capabilities; performance differences between projects are substantial (e.g., P2 test-wise Pass@1 near 0% for multiple models)."
     },
     {
       "flag": "LLM coverage excludes frontier models",
       "detail": "Only 5 LLMs are evaluated, with GPT-3.5 as the largest closed-source model; GPT-4, Claude, and Gemini are absent due to resource constraints, limiting the benchmark's ability to characterize the full capability range."
     },
     {
       "flag": "No technical contamination resistance",
       "detail": "Contamination mitigation relies entirely on academic confidentiality of student assignments (2019-2022), with no canary strings, temporal splits, or dynamic generation — the confidentiality claim is unverifiable."
     },
     {
       "flag": "No funding disclosure",
       "detail": "The paper includes no acknowledgments, funding statement, or grant information, making it impossible to assess potential conflicts of interest."
     },
     {
       "flag": "Benchmark license unspecified",
       "detail": "The paper provides a GitHub URL for the benchmark but does not specify any license, leaving reuse terms legally ambiguous."
     }
   ],
   "cited_papers": [
     {
       "title": "HumanEval: Evaluating Large Language Models Trained on Code",
       "relevance": "The primary baseline code generation benchmark that JavaBench is compared against and positioned to extend to project-level Java OOP."
     },
     {
       "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation",
       "relevance": "The most directly comparable predecessor benchmark at class-level Python; JavaBench extends to project-level Java and adopts similar synthesis strategy designs."
     },
     {
       "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
       "relevance": "Another project-level Python benchmark that JavaBench is compared against in Table 1 to establish the gap in Java project-level evaluation."
     },
     {
       "title": "OOP: Object-Oriented Programming Evaluation Benchmark for Large Language Models",
       "relevance": "The only prior benchmark claiming to test OOP features, which JavaBench critiques for not providing actual code context — only OOP concepts in prompts."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "relevance": "A key RAG-based approach for project-level code completion that the selected context design partially complements."
     },
     {
       "title": "CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion",
       "relevance": "A multilingual benchmark including Java that JavaBench is distinguished from by providing project-level rather than statement-level evaluation."
     },
     {
       "title": "Time Travel in LLMs: Tracing Data Contamination in Large Language Models",
       "relevance": "Referenced as the basis for contamination concerns that JavaBench attempts to mitigate through academic confidentiality."
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "relevance": "Supports the motivation for selective context design — finding that LLMs fail to use information in the middle of long contexts, relevant to JavaBench's context setting ablation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Engineers choosing LLMs for Java enterprise development and researchers designing prompting strategies can directly apply the context selection and synthesis strategy findings."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that method signatures only (not full context) outperforms providing maximum context is counterintuitive and practically actionable; the severity of the LLM-student gap on what seems like a student-grade assignment is striking."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns are raised; the paper focuses on capability evaluation with no threat modeling."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The LLMs-vs-undergraduates framing has mild drama potential, but the domain (student Java assignments) limits headline appeal."
     },
     "demo_ability": {
       "score": 3,
       "justification": "A public leaderboard (java-bench.github.io/leaderboard.html) and GitHub repository allow anyone to immediately test their model and compare results."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "HKUST and Nanjing University are reputable academic institutions but not AI-brand-name labs; no famous product or company association."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "39483482",
         "title": "Show HN: OK-Robot: open, modular home robot framework for pick-and-drop anywhere",
         "points": 542,
         "comments": 110,
         "url": "https://news.ycombinator.com/item?id=39483482"
       },
       {
         "hn_id": "36475563",
         "title": "AudioPaLM: A large language model that can speak and listen",
         "points": 69,
         "comments": 11,
         "url": "https://news.ycombinator.com/item?id=36475563"
       },
       {
         "hn_id": "40727755",
         "title": "Adversarial Perturbations Cannot Reliably Protect Artists from Generative AI",
         "points": 5,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40727755"
       },
       {
         "hn_id": "40755630",
         "title": "Adversarial Perturbations Cannot Reliably Protect Artists from Generative AI",
         "points": 4,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40755630"
       },
       {
         "hn_id": "41617735",
         "title": "WaveletGPT: Wavelets Meet Large Language Models",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41617735"
       },
       {
         "hn_id": "39764168",
         "title": "A tweezer array with 6100 highly coherent atomic qubits",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39764168"
       },
       {
         "hn_id": "40748080",
         "title": "Adversarial Perturbations Cannot Reliably Protect Artists from Generative AI",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40748080"
       },
       {
         "hn_id": "27612994",
         "title": "LegoFormer: Transformers for Block-by-Block Multi-View 3D Reconstruction",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=27612994"
       },
       {
         "hn_id": "40855651",
         "title": "Generalist Lightweight Model for Various Information Extraction Tasks",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40855651"
       }
     ],
     "top_points": 542,
     "total_points": 632,
     "total_comments": 122
   }
 }