ai-research-survey

scan-v4.json (31220B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "DeepSeek-Coder: When the Large Language Model Meets Programming — The Rise of Code Intelligence",
     "authors": [
       "Guo, D.",
       "Zhu, Q.",
       "Yang, D.",
       "Xie, Z.",
       "et al."
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2401.14196",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that DeepSeek-Coder 'achieves state-of-the-art performance among open-source code models' and 'surpasses existing closed-source models like Codex and GPT-3.5' are supported by Tables 3-8. The claim about surpassing GPT-3.5 is hedged with 'majority of the evaluation benchmarks.'",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are primarily from ablation studies: FIM rate effect (Figure 3, controlled single-variable manipulation) and repository-level pre-training effect (Table 7, 'w/o Repo Pre-training' comparison). These ablations involve controlled manipulation of single variables.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'When the Large Language Model Meets Programming — The Rise of Code Intelligence' is extremely broad. The subtitle suggests a general claim about code intelligence, while results are on specific benchmarks. The paper does not explicitly bound its generalization claims to the tested benchmarks and languages.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for performance improvements. The paper attributes gains to data quality and training methodology but does not consider whether larger training data volume, different tokenization, or other factors could explain the results.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames benchmark performance (Pass@1 on HumanEval, MBPP) as evidence of 'code intelligence' and 'programming' capability without discussing the gap between benchmark performance and real-world coding ability. The title and introduction frame this broadly as 'code intelligence' but measurements are narrow benchmark metrics.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No limitations or threats-to-validity section. The conclusion mentions future work on long-context but does not discuss study limitations.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does briefly acknowledge contamination risk for LeetCode (Section 4.1) but this is not a systematic discussion of threats.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what its benchmark results do or do not show about real-world coding ability.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure. The paper is from DeepSeek-AI but does not mention any funding source or grants.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: DeepSeek-AI and Peking University. The paper is evaluating DeepSeek-AI's own models.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "DeepSeek-AI is evaluating its own product. The company has a direct commercial interest in demonstrating strong performance for its models. This conflict is not acknowledged.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. DeepSeek-AI authors evaluating DeepSeek-Coder models have obvious financial interests in positive results, but this is not declared.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Fill-in-the-Middle (FIM), PSM/SPM modes, repository-level pre-training, and evaluation benchmarks are all defined or cited to prior work with clear explanations.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four explicit contributions are listed in the introduction: the model series, first repository-level pre-training, FIM strategy analysis, and benchmark evaluations.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper situates DeepSeek-Coder against StarCoder, CodeLlama, CodeGeeX2, and Codex, and builds directly on prior FIM work (Bavarian et al.) and deduplication research (Lee et al., Kocetkov et al.).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "GitHub repository URL provided: https://github.com/deepseek-ai/DeepSeek-Coder. The LeetCode benchmark is also released at the same repository.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The training dataset (798GB of source code) is not released. The paper describes the data collection pipeline but does not provide a download link for the training corpus. Evaluation benchmarks used are public (HumanEval, MBPP, DS-1000, CrossCodeEval).",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions using HAI-LLM framework, NVIDIA A100 and H800 GPUs, and FlashAttention v2, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions for reproduction.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided for training or evaluation. The paper describes the methodology at a high level but lacks specific commands or scripts to reproduce results.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results in Tables 3-10 are reported as point estimates (e.g., '56.1% accuracy') with no confidence intervals or error bars.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims DeepSeek-Coder 'surpasses' and 'outperforms' various baselines based solely on comparing numbers in tables, without any statistical significance tests.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports absolute performance numbers with baseline context, e.g., 'our model has demonstrated a notable improvement of 9% and 11% in accuracy' compared to CodeLlama-Base 34B on HumanEval and MBPP respectively (Section 4.1). Per-benchmark breakdowns provide sufficient context.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification for benchmark sample sizes. The LeetCode benchmark uses 180 problems and the paper does not discuss whether this is sufficient for the claims made.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance or standard deviation is reported across experimental runs. All results appear to be single-run greedy decoding numbers. The paper states 'we adopted a greedy search approach' but does not report variance across seeds or runs.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Comprehensive baselines included: CodeGeeX2, StarCoder, CodeLlama (7B/13B/34B), code-cushman-001, GPT-3.5, GPT-4 across all benchmarks (Tables 3-8).",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include contemporaneous models: CodeLlama (2023), StarCoder (2023), GPT-4 (2023), and GPT-3.5-Turbo. These were state-of-the-art at time of writing.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "FIM ablation study in Section 3.1.2 compares 0% FIM, 50% FIM, 100% FIM, and 50% MSP rates on the 1.3B model, with results in Figure 3. Section 4.3 Table 7 includes ablation of repository-level pre-training ('w/o Repo Pre-training').",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics used: Pass@1 on HumanEval and MBPP, exact match (EM) and edit similarity (ES) on CrossCodeEval, per-library accuracy on DS-1000, and difficulty-stratified results on LeetCode.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "No human evaluation of code quality. All evaluation is automated via test case execution (Pass@1, exact match, etc.).",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Results are reported on established test sets (HumanEval, MBPP, DS-1000, CrossCodeEval). The LeetCode benchmark uses problems from July 2023-January 2024, after the training data cutoff of February 2023.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Extensive breakdowns: per-language results on HumanEval (Table 3), per-library results on DS-1000 (Table 4), per-difficulty on LeetCode (Table 5), per-language on CrossCodeEval (Table 7).",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No error analysis or failure case discussion. The paper does not examine what types of problems the model fails on or why.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The FIM ablation (Figure 3) shows that 100% FIM rate results in 'weakest code completion capability,' demonstrating a trade-off. Section 5 notes DeepSeek-Coder-v1.5 has 'a slight decrease in coding performance' compared to the base model (Table 10). The paper also acknowledges a 'substantial performance gap' vs GPT-4.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "For their own models, sizes are specified (1.3B, 6.7B, 33B). For baselines, GPT-3.5-Turbo and GPT-4 are referenced without snapshot dates or API versions. code-cushman-001 is named but other baseline versions (e.g., which CodeLlama checkpoint) lack precision.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "The LeetCode prompt template is provided verbatim: '{problem_description}\\nPlease complete the code below to solve the above problem:\\n```python\\n{code_template}\\n```'. The CoT addition is also quoted. FIM format with sentinel tokens is specified. HumanEval/MBPP use standard zero-shot/few-shot formats.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 provides detailed hyperparameters (hidden size, layers, attention heads, batch size, learning rate) for all model sizes. Section 3.4 specifies AdamW optimizer with β1=0.9, β2=0.95, warm-up steps, learning rate schedule. Section 3.7 gives instruction tuning parameters.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The models are evaluated directly on benchmark tasks with standard prompting.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 2 provides detailed data preprocessing: filtering rules from StarCoder (line length, alphabetic character ratio, XML filtering, HTML visible text ratio, JSON/YAML size limits), dependency parsing (Algorithm 1), repo-level deduplication (Section 2.3), quality screening with compiler and quality model (Section 2.4). Table 1 shows final data statistics.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The training data is not released. Evaluation outputs (model generations) are not provided for independent verification.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 2 describes data collection in detail: public GitHub repositories created before February 2023, 87 programming languages, filtering rules, deduplication, quality screening. Table 1 provides statistics for all 87 languages.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data sources are public GitHub repositories and standard benchmarks.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Figure 2 and Section 2 document the full pipeline: Data Crawling → Rule Filtering (reduced to 32.8% of original) → Dependency Parsing → Repo-Level Deduplication → Quality Screening. Final statistics in Table 1 (798GB, 603M files).",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 states 'We collect public repositories created before February 2023 on GitHub.' This establishes a clear training data cutoff.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.4 describes n-gram decontamination: 'we filter out files containing docstrings, questions, and solutions from sources such as HumanEval, MBPP, GSM8K and MATH' using 10-gram and exact match filtering. Section 4.1 also acknowledges potential contamination in LeetCode results.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.4 implements active decontamination for HumanEval, MBPP, GSM8K, and MATH via n-gram filtering. Section 4.1 acknowledges 'the possibility of data contamination cannot be entirely ruled out' for LeetCode and notes higher scores in July-August contests. CrossCodeEval uses repos from March-June 2023, after the training cutoff.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost, latency, or tokens consumed are reported for any evaluation.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Hardware is mentioned (A100, H800 GPUs) but total GPU hours, training time, or compute budget are not quantified. Training 33B parameters on 2T tokens is massive compute but unquantified.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "No multi-seed results reported. All benchmark results appear to be single-run with greedy decoding.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The number of evaluation runs is not stated. Greedy decoding implies single-run but this is not explicit.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "The FIM ablation tests 4 configurations but no broader hyperparameter search budget is reported for training or evaluation settings.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": true,
           "justification": "The FIM rate selection is justified: 50% PSM chosen 'to achieve a balance between FIM efficiency and code completion proficiency' based on ablation results showing the trade-off (Section 3.1.2, Figure 3).",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": false,
           "answer": false,
           "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors evaluate their own DeepSeek-Coder models against baselines they re-implemented ('re-implemented the baseline results using the same script'). Self-comparison bias is not acknowledged.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": false,
           "justification": "DeepSeek-Coder is trained on 2T tokens while baselines use different amounts (CodeLlama uses 500B). This compute difference is not discussed in relation to performance gains.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses HumanEval, MBPP, etc. without questioning whether they measure 'code intelligence.' It briefly notes HumanEval/MBPP 'rely heavily on straightforward programming tasks' but introduces DS-1000 as a fix rather than discussing construct validity broadly.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No scaffolding is used; models are evaluated directly on benchmarks.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Training data cutoff is February 2023. The LeetCode benchmark uses problems from July 2023-January 2024. CrossCodeEval uses repos from March-June 2023. Both are explicitly noted as post-training-cutoff.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether evaluation setup leaks information. For example, no discussion of whether HumanEval docstrings or function signatures contain answer-leaking information.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether training data and test benchmarks share structural similarities (e.g., similar coding patterns, same repositories).",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.4 describes a concrete n-gram filtering decontamination method: 10-gram matching for long strings, exact match for 3-9 gram strings, applied against HumanEval, MBPP, GSM8K, and MATH test data.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DeepSeek-Coder-Base 33B achieves state-of-the-art performance among open-source code models across HumanEval, MBPP, DS-1000, and FIM benchmarks.",
       "evidence": "Tables 3, 4, and 6 show DeepSeek-Coder-Base 33B outperforming StarCoder-16B, CodeLlama-34B, and CodeGeeX2-6B on all tested benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek-Coder-Instruct 33B surpasses GPT-3.5-Turbo on most coding benchmarks.",
       "evidence": "Table 3 shows 69.2% vs 64.9% average on multilingual HumanEval; Table 5 shows 27.8% vs 23.3% on LeetCode Contest.",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek-Coder-Base 6.7B achieves competitive performance with CodeLlama-Base 34B despite having 5x fewer parameters.",
       "evidence": "Table 3: 44.7% vs 41.0% average HumanEval; Table 4: 30.5% vs 34.3% DS-1000 (slightly below); Table 8: 54.7% vs 62.0% math avg — mixed evidence.",
       "supported": "moderate"
     },
     {
       "claim": "Repository-level pre-training significantly boosts cross-file code generation capability.",
       "evidence": "Table 7 shows consistent drops when removing repo pre-training (e.g., Java: 17.72% → 16.64% EM with retrieval), though improvements are modest (1-2pp).",
       "supported": "moderate"
     },
     {
       "claim": "50% PSM FIM rate is the optimal training policy balancing FIM and code completion performance.",
       "evidence": "Figure 3 ablation across 0%, 50% PSM, 100% PSM, and 50% MSP shows 50% PSM as best overall trade-off on HumanEval + HumanEval-FIM + MBPP.",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek-Coder-v1.5 maintains coding performance while substantially improving math and natural language understanding.",
       "evidence": "Table 10: HumanEval drops 1.5pp (44.7% → 43.2%) while GSM8K improves 19.2pp (43.2% → 62.4%) and MMLU improves 12.5pp (36.6% → 49.1%).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DeepSeek-Coder-Base 33B achieves state-of-the-art performance among open-source code LLMs on HumanEval, MBPP, DS-1000, CrossCodeEval, and program-aided math reasoning benchmarks, trained from scratch on 2T tokens with repository-level organization and 50% PSM Fill-in-the-Middle pre-training. DeepSeek-Coder-Instruct 33B surpasses GPT-3.5-Turbo on multilingual HumanEval (69.2% vs 64.9%) and LeetCode Contest (27.8% vs 23.3%), substantially narrowing the gap to GPT-4. Ablations confirm that repository-level pre-training modestly improves cross-file completion and that FIM rate involves a trade-off between infilling and autocompletion performance. Continuing pre-training from a general-purpose base (DeepSeek-LLM-7B) preserves coding performance while dramatically improving math reasoning (+19pp on GSM8K) and natural language understanding.",
   "red_flags": [
     {
       "flag": "No variance across runs",
       "detail": "All benchmark results are single point estimates with no standard deviation, confidence intervals, or multiple evaluation seeds — small differences (1-3pp) cannot be distinguished from noise."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section; the only acknowledgment of limitations is one sentence about LeetCode contamination."
     },
     {
       "flag": "Baseline model versions unspecified",
       "detail": "GPT-3.5-Turbo and GPT-4 are compared without snapshot dates or API version identifiers, making comparisons unreproducible as these models change over time."
     },
     {
       "flag": "Self-evaluation without independent replication",
       "detail": "DeepSeek-AI employees evaluate their own model with no independent verification; conflict of interest is not disclosed and no competing interests statement is provided."
     },
     {
       "flag": "LeetCode contamination unresolved",
       "detail": "The paper acknowledges higher scores in July-August contests and that contamination 'cannot be entirely ruled out' but provides no quantitative analysis of contamination impact."
     },
     {
       "flag": "Training data not released",
       "detail": "The 798GB training corpus is described but not released, making the most important methodological contribution (data pipeline and quality) unverifiable."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (Codex / HumanEval)",
       "relevance": "Introduces the HumanEval benchmark used as the primary evaluation throughout and establishes the code LLM evaluation paradigm."
     },
     {
       "title": "StarCoder: May the Source Be With You",
       "relevance": "Direct competitor and baseline; DeepSeek-Coder adopts and extends StarCoder's data filtering rules and FIM approach."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "relevance": "Primary open-source competitor used as baseline across all evaluations; DeepSeek-Coder claims to outperform it with fewer parameters."
     },
     {
       "title": "Efficient Training of Language Models to Fill in the Middle",
       "relevance": "Foundation for the FIM pre-training approach adopted by DeepSeek-Coder; the PSM/SPM modes are built on this work."
     },
     {
       "title": "CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion",
       "relevance": "Key evaluation benchmark for cross-file completion; used to validate the repository-level pre-training contribution."
     },
     {
       "title": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation",
       "relevance": "Evaluation benchmark for practical data science code generation across 7 libraries, addressing HumanEval's focus on simple algorithmic tasks."
     },
     {
       "title": "Program Synthesis with Large Language Models (MBPP)",
       "relevance": "Second primary evaluation benchmark; used throughout to measure Python code generation quality."
     },
     {
       "title": "The Stack: 3 TB of Permissively Licensed Source Code",
       "relevance": "Prior dataset work that DeepSeek-Coder builds upon for data filtering methodology and near-deduplication approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Open-source code models with permissive licensing that developers can immediately use for code completion and generation across 87 languages."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The 6.7B matching 34B CodeLlama is mildly surprising but the overall narrative of 'our model beats baselines' is standard."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or misuse concerns are discussed or raised by the work."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly challenges Meta's CodeLlama dominance and claims to beat GPT-3.5, but framed cooperatively rather than confrontationally."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Models are publicly available on HuggingFace with a GitHub repo, pip-installable via standard HF tooling, and ready to use immediately."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "DeepSeek became widely recognized in the AI community, though at time of publication it was still building its reputation compared to OpenAI or Meta."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "39142278",
         "title": "Python has 189X the dataset size compared to Rust",
         "points": 2,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=39142278",
         "created_at": "2024-01-26T13:18:01Z"
       }
     ],
     "top_points": 2,
     "total_points": 2,
     "total_comments": 4
   }
 }