ai-research-survey

scan-v5.json (30412B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Holistic Evaluation of State-of-the-Art LLMs for Code Generation",
     "authors": [
       "Le Zhang",
       "Suresh Kothari"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.18131",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All claims in abstract (evaluation of 6 LLMs on 944 LeetCode problems across 5 languages, metric categories, DeepSeek-R1/GPT-4.1 outperformance) are demonstrated in results sections with detailed tables and figures.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Paper makes minimal causal claims. Section 4.6 tests prompt engineering effect quasi-experimentally on 202 problems, showing optimization hints reduce algorithmic suboptimality 2-5% across models.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Paper evaluates only LeetCode algorithmic problems but generalizes conclusions to 'real-world software development tasks' and 'software engineering' broadly. LeetCode is a narrow, artificial benchmark that doesn't represent production code modification, security-critical systems, or maintainability scenarios.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "When Llama-3.3 underperforms, paper doesn't explore whether this is due to: (a) inherent model weakness, (b) poor prompt fit, (c) suboptimal hyperparameters, or (d) task mismatch. Single explanations presented without alternatives.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1 directly measures 'percentage of problems solved correctly on first attempt.' Metrics (CE, RE, FF, AS) are direct measurements mapped to explicit definitions; claims about 'correctness' and 'efficiency' align with metric granularity.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated 'Limitations' or 'Threats to Validity' section. Contamination concern mentioned in Section 3.2 and future work in Section 6, but no systematic review of study limitations.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Contamination mitigation ('selected recent problems', 'avoided discussion content') is specific. But systematic threats—selection bias in 944 problems, prompt format bias, LeetCode's artificial time limits vs. real-world constraints—are not addressed.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Paper states what it evaluates ('944 LeetCode problems, 5 languages') but does NOT explicitly state scope boundaries—what results do NOT show (e.g., no claim about real-world code modification, security, maintainability, multi-turn interactions). Future work section partially addresses this.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement anywhere in the paper. Authorship shows Iowa State University affiliation, but no disclosure of grant support, sponsorship, or funding source.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors listed as 'Department of Computer Science, Iowa State University, Ames, Iowa, USA.' No apparent financial ties to evaluated model companies (OpenAI, Anthropic, DeepSeek, Meta, Alibaba).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Iowa State is independent of LLM vendors. No evidence of company funding or incentive to bias results toward any model.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No 'Competing Interests' or 'Conflicts of Interest' statement. No disclosure of patents, equity stakes, consulting arrangements, or financial relationships.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms operationalized through metrics: 'correctness' (Pass@1), 'efficiency' (algorithmic suboptimality), 'robustness' (runtime errors). Metrics are formally defined in Section 3.6.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit objectives stated in Introduction: (1) evaluate coding proficiency across languages, (2) benchmark strengths/weaknesses, (3) provide practical guidance. Contributions are unambiguous.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 reviews prior work (Chen et al. 2021, Pearce et al. 2025, Nijkamp et al. 2022). Section 3.2 explicitly contrasts: 'Previous studies [Döderlein, Coignion] are based on older models... Our study provides more comprehensive and up-to-date evaluation.'",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.7 states: 'All the artifacts of this study, including our datasets, code, and evaluation results, are available in a public repository anonymously: https://figshare.com/s/26448e92798aab34e407'",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "944 LeetCode problems and their results are publicly released via figshare. LeetCode problems are from a public platform. Raw submission reports available.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, Python version specs, or dependency list provided. Paper mentions 'custom LeetCode API' but does not document its environment or dependencies.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Experimental setup is described (prompt format, hyperparameters, model names) but no step-by-step instructions to run the pipeline. How to access figshare, invoke the API, generate solutions for all 6 models, and reproduce results is not documented.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results reported as single-point percentages (e.g., 'DeepSeek-R1 Python: 89.30%'). No confidence intervals, error bars, standard deviations, or uncertainty quantification.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Comparative claims ('DeepSeek-R1 consistently outperforms'; 'Llama-3.3 shows lowest Pass@1') are presented without statistical significance tests. Differences could be within noise.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Percentages and absolute differences (e.g., DeepSeek-R1 89.3% vs Llama-3.3 54.8% on Python) are reported. These are effect sizes, though not standardized or confidence-bounded.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "944 problems for large dataset, 202 for small dataset. No power analysis, justification for these counts, or discussion of why these sample sizes are adequate.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results are single-point estimates per model-language combination. No variance, standard deviation, run-to-run variation, or multiple trials. Pass@1 is deterministic per model run.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Six models are evaluated against each other, providing pairwise comparisons. Each model serves as a baseline for others.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All six models are from 2024-2025 releases. Includes general-purpose (Llama-3.3, Claude-3.7) and code-specialized models (Qwen2.5-Coder, DeepSeek-Coder variants). Contemporaneous and representative.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.6 provides one ablation: optimization prompt ('Optimize the time complexity') tested on 202 problems across all models. Shows 2-5% improvement in algorithmic suboptimality for most models.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Five quality metrics (Pass@1, CE, RE, FF, AS) plus token usage and cost analysis. Multiple dimensions of code quality evaluated.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects; LeetCode's automated test cases are the evaluation criterion. Not applicable.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "LeetCode provides extensive test cases per problem (7–9,558 cases, mean varies by problem). Models are evaluated against these unseen test cases on submission.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results broken down by model (6 models), language (5 languages), difficulty tier (Easy/Medium/Hard in Table 2), and metric. Per-language and per-model analysis in Figures 2–9.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Abstract mentions 'common failure scenarios such as syntax errors, logical flaws, and suboptimal algorithms,' but no specific case studies, examples, or detailed failure analysis provided in results.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "All models' results reported fully, including poor performance (e.g., Llama-3.3's 54.8% Pass@1 on Python, 41.6% algorithmic suboptimality on JavaScript). No selective reporting.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Table 1 lists model names and sizes, but lacks exact snapshot dates for proprietary models. GPT-4.1 and Claude-3.7 are referenced to 2025 announcements but no API version or access date specified. Open-source models are better identified (e.g., Llama-3.3-70B-Instruct).",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Figure 1 shows a complete, concrete prompt example with all components: role specification, problem description, constraints, code snippet template, test cases, and instructions. Not a template with placeholders.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.3 specifies: top-p = 0.95, temperature = 0.1 for all models. Rationale provided (prior research showing these values optimize Pass@1).",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Prompt scaffolding (role, problem, constraints, code snippet, test cases, instructions) is detailed in Section 3.4 with subsections 3.4.1–3.4.5 and a full example.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "Problems are described as '944 manually picked' from LeetCode and '202 selected... based on their sensitivity to algorithmic complexity,' but selection criteria and preprocessing steps are not documented.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.7 states data and results are available in the public figshare repository. Assuming the repository contains the promised datasets and submission results.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": false,
           "justification": "Data source is clear (LeetCode), but 'manually picked' criteria are undefined. What makes a problem worthy of selection? How were 944 chosen from thousands? No documented procedure.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; benchmark is standard. Not applicable.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "High-level pipeline described (select problems, create prompts, submit to LeetCode API, collect results) but detailed steps missing: How are submission reports parsed? How are metrics computed from reports? No pseudocode or detailed algorithm.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Section 3.2 discusses contamination risk but does not state training data cutoff dates for any model. DeepSeek-R1 (2025), GPT-4.1 (2025), Claude-3.7 (2025)—exact cutoffs unknown.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 explicitly addresses: 'possibility that some problems or their corresponding solutions may have appeared in the pretraining corpora.' Mitigation strategy: 'intentionally selected relatively recent problems... avoided using any LeetCode discussion content.'",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination is acknowledged as a risk. Mitigations are described (recent problem selection, no discussion forum content, unseen task treatment). However, 'complete elimination of contamination cannot be guaranteed,' per the text.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects. Not applicable.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects. Not applicable.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects. Not applicable.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects. Not applicable.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects. Not applicable.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects. Not applicable.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects. Not applicable.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.7.2 reports total costs in USD per model: DeepSeek-R1 (regular $81.75, off-peak $20.52), GPT-4.1 ($44.41), Claude-3.7 ($108.65), etc. Also Figure 8 shows token usage breakdown.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.7 reports token consumption (input and output tokens per model) and API costs. DeepSeek-R1 uses 36.52M output tokens; GPT-4.1 uses 4.74M. Cost data implicitly captures compute budget.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DeepSeek-R1 and GPT-4.1 consistently outperform other models in correctness, efficiency, and robustness",
       "evidence": "Pass@1 scores: DeepSeek-R1 86.97–89.30% across languages; GPT-4.1 83.26–86.55%. Algorithmic suboptimality (AS): DeepSeek-R1 1.38–2.23%; GPT-4.1 2.54–4.77%. Runtime errors (RE): GPT-4.1 lowest across most languages.",
       "supported": "strong"
     },
     {
       "claim": "Algorithmic suboptimality is common, especially in Llama-3.3, which defaults to brute-force solutions",
       "evidence": "Llama-3.3 AS: 7.63–10.49% vs. DeepSeek-R1 1.38–2.23%. Figure 6 shows Llama-3.3 consistently highest AS across all 5 languages. No direct evidence of 'brute-force' reasoning, only outcome-level observation.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt engineering with optimization hints significantly improves algorithmic efficiency",
       "evidence": "Section 4.6: Adding 'Optimize the time complexity' hint reduces AS 2–5% for most models on 202-problem small dataset. DeepSeek-R1 shows consistent improvement across all languages (2–4% reduction).",
       "supported": "moderate"
     },
     {
       "claim": "Python3 and JavaScript yield fewer compile and runtime errors across all models, while stricter languages like C++ and Go reveal more limitations",
       "evidence": "Figure 3 (CE): Python and JavaScript show 0% CE for all models; Go shows 2.01–7.10% CE. Figure 4 (RE): Go shows strong performance; JavaScript shows higher RE (1.91–4.34%) than Go. Language difficulty gradient is clear.",
       "supported": "strong"
     },
     {
       "claim": "Larger or more specialized models tend to perform better in programming evaluations",
       "evidence": "Larger models (DeepSeek-R1 671B, GPT-4.1 1800B) show higher Pass@1 (86–89%) than Llama-3.3 (52–55%). Qwen2.5-Coder (32B, code-specialized) outperforms Llama-3.3 (70B, general-purpose) in Pass@1 and FF metrics.",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek-R1's reasoning capabilities provide advantages in efficiency-sensitive tasks compared to DeepSeek-V3",
       "evidence": "DeepSeek-R1 AS: 1.38–2.23%; DeepSeek-V3 AS: 2.65–3.28%. DeepSeek-R1 explicitly designed for 'complex reasoning.' Figure 6 shows consistent advantage across languages.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt responsiveness to optimization cues varies significantly across models; Llama-3.3 shows minimal improvement despite starting from worst baseline",
       "evidence": "Figure 7 (optimization hint experiment): DeepSeek-R1 reduces AS 1.98–3.47pp; Llama-3.3 reduces only 1.49–2.49pp despite starting at 40–44% AS. Claude-3.7 shows nearly no improvement.",
       "supported": "strong"
     },
     {
       "claim": "Successful LLM deployment in software engineering requires human oversight, not autonomous code generation",
       "evidence": "Conclusions (5.1, 5.2): 'Even top models occasionally fail on complex problems or edge cases, underscoring the need for human oversight.' Recommendations: 'Treat LLM-generated code as a first draft. Always review for logical correctness, edge cases, and efficiency.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "This paper evaluates six state-of-the-art LLMs on 944 LeetCode problems across five programming languages, using five quality metrics. DeepSeek-R1 and GPT-4.1 achieve 86–89% Pass@1 (first-attempt correctness), while Llama-3.3 achieves only 52–55%. Models are most proficient in dynamically-typed languages (Python, JavaScript) and struggle with strict syntactic constraints in Go and C++. Algorithmic efficiency is a widespread weakness: Llama-3.3 shows 7.6–10.5% time-limit exceeded errors, indicating brute-force solutions. Prompt engineering with optimization hints improves performance modestly (2–5%), but responsiveness varies by model. DeepSeek-R1's reasoning-optimized design provides measurable efficiency advantages over its non-reasoning variant. Even top models fail on edge cases, underscoring the need for human code review.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All results reported as single-point percentages without confidence intervals, p-values, or variance estimates. Claimed differences (e.g., 2–3pp between models) may be within measurement noise on a 944-problem sample."
     },
     {
       "flag": "Overgeneralization from benchmark to real-world",
       "detail": "LeetCode problems are algorithmic, isolated, and artificial. Conclusions extrapolate to 'real-world software development,' ignoring code maintenance, security, refactoring, and multi-turn interactions absent from the evaluation."
     },
     {
       "flag": "No dedicated limitations section",
       "detail": "Methodological threats (selection bias in 944 problems, LeetCode time-limit realism, contamination risk) are scattered or unaddressed. No explicit statement of what results do NOT show."
     },
     {
       "flag": "Contamination risk unresolved",
       "detail": "Paper acknowledges LeetCode problems may have appeared in training corpora but provides no empirical evidence that 'recent problem' selection eliminated this risk. Training cutoff dates not disclosed for any model."
     },
     {
       "flag": "Limited reproducibility documentation",
       "detail": "While code and data are released via figshare, no step-by-step reproduction instructions, environment specifications, or Dockerfile provided. Custom LeetCode API is undocumented."
     },
     {
       "flag": "Insufficient data collection transparency",
       "detail": "944 problems are 'manually picked' with undefined criteria. How were these selected from thousands? What selection bias might this introduce?"
     },
     {
       "flag": "Minimal ablation study",
       "detail": "Only one ablation (optimization prompt) tested on 202 problems. Other design choices (hyperparameters, prompt structure) not systematically tested."
     },
     {
       "flag": "No funding or conflict-of-interest disclosure",
       "detail": "Paper includes no statement of funding source or competing interests. No acknowledgment of financial relationships with model vendors."
     },
     {
       "flag": "Lack of failure case analysis",
       "detail": "Mentions 'syntax errors, logical flaws, suboptimal algorithms' as failure types but provides zero case studies, code excerpts, or detailed failure analysis to ground findings."
     },
     {
       "flag": "Model version ambiguity for proprietary models",
       "detail": "GPT-4.1 and Claude-3.7 lack exact API version, snapshot date, or access timing. Results may be sensitive to model updates within 2025."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "relevance": "Foundational benchmark study (Chen et al. 2021, Codex evaluation); prior work on LLM code generation methodologies"
     },
     {
       "title": "A Performance Study of LLM-Generated Code on LeetCode",
       "relevance": "Direct precedent for LeetCode-based evaluation; prior hyperparameter study (Coignion et al. 2024)"
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "relevance": "Multi-turn code generation paradigm; contrasts with this paper's Pass@1 single-shot evaluation (Nijkamp et al. 2022)"
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "relevance": "Security vulnerabilities in LLM-generated code; related evaluation dimension (Pearce et al. 2025)"
     },
     {
       "title": "Proving the Coding Interview: A Benchmark for Formally Verified Code Generation",
       "relevance": "Formal verification of LLM-generated code; alternative evaluation criterion beyond runtime testing (Dougherty & Mehta 2025)"
     },
     {
       "title": "The Curious Case of Neural Text Degeneration",
       "relevance": "Foundational work on top-p and temperature hyperparameters; cited for decoding parameter justification (Holtzman et al. 2019)"
     },
     {
       "title": "LeetCodeDataset: A Temporal Dataset for Robust Evaluation and Efficient Training of Code LLMs",
       "relevance": "Contemporary temporal dataset for code LLM evaluation; time-aware benchmark construction (Xia et al. 2025)"
     },
     {
       "title": "A Survey on Large Language Models for Code Generation",
       "relevance": "Comprehensive survey of NL2Code paradigm and code generation capabilities (Jiang et al. 2024)"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Practitioners evaluating LLM adoption for code generation get direct model comparisons, cost analysis, and per-language performance. Actionable for tool selection and prompt design."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Results confirm conventional expectations: larger/specialized models outperform smaller ones, strict languages are harder, prompt engineering helps. No counterintuitive findings."
     },
     "fear_safety": {
       "score": 0,
       "justification": "Paper does not address security vulnerabilities, alignment, or AI safety concerns. Focuses on code correctness only, not risk."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward benchmark study with no controversy or competing narratives. Clear winner (DeepSeek-R1) without debate."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Results highly reproducible: select any LeetCode problem, run six models via free/paid APIs, compare outputs. Readers can verify findings immediately."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Iowa State University is reputable but not a household name. Study is arXiv preprint, not published in top-tier venue (ICLR, NeurIPS, ACL). Authors less known."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "42557775",
         "title": "Mulberry: Empowering MLLM with o1-like Reasoning",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42557775",
         "created_at": "2024-12-31T10:22:36Z"
       },
       {
         "hn_id": "46425525",
         "title": "Optimal Software Pipelining and Warp Specialization for Tensor Core GPUs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46425525",
         "created_at": "2025-12-29T20:54:07Z"
       },
       {
         "hn_id": "46069881",
         "title": "Conformal Prediction for Compositional Data",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46069881",
         "created_at": "2025-11-27T15:03:53Z"
       },
       {
         "hn_id": "46363503",
         "title": "Layout-Aware Text Editing for Efficient Conversion of Academic PDFs to Markdown",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46363503",
         "created_at": "2025-12-23T08:26:53Z"
       },
       {
         "hn_id": "43150514",
         "title": "Intuitive physics understanding emerges from self-supervised pretraining",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43150514",
         "created_at": "2025-02-23T16:27:24Z"
       },
       {
         "hn_id": "46021507",
         "title": "World-in-World: World Models in a Closed-Loop World",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46021507",
         "created_at": "2025-11-23T07:25:35Z"
       }
     ],
     "top_points": 3,
     "total_points": 10,
     "total_comments": 1
   }
 }