ai-research-survey

scan.json (29041B)

---

 {
   "paper": {
     "title": "DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence",
     "authors": [
       "Qihao Zhu",
       "Daya Guo",
       "Zhihong Shao",
       "Dejian Yang",
       "Peiyi Wang",
       "Runxin Xu",
       "Y. Wu",
       "Yukun Li",
       "Huazuo Gao",
       "Shirong Ma",
       "Wangding Zeng",
       "Xiao Bi",
       "Zihui Gu",
       "Hanwei Xu",
       "Damai Dai",
       "Kai Dong",
       "Liyue Zhang",
       "Yishi Piao",
       "Zhibin Gou",
       "Zhenda Xie",
       "Zhewen Hao",
       "Bingxuan Wang",
       "Junxiao Song",
       "Deli Chen",
       "Xin Xie",
       "Kang Guan",
       "Yuxiang You",
       "Aixin Liu",
       "Qiushi Du",
       "Wenjun Gao",
       "Xuan Lu",
       "Qinyu Chen",
       "Yaohui Wang",
       "Chengqi Deng",
       "Jiashi Li",
       "Chenggang Zhao",
       "Chong Ruan",
       "Fuli Luo",
       "Wenfeng Liang"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2406.11931"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DeepSeek-Coder-V2, a 236B MoE model (21B active params), achieves performance comparable to GPT-4 Turbo on code and math benchmarks including 90.2% on HumanEval, 76.2% on MBPP+, 75.7% on MATH, and 43.4% on LiveCodeBench. The 16B variant (2.4B active) outperforms larger dense models like DeepSeek-Coder-33B. The model supports 338 programming languages and 128K context length. A 1B ablation study validates the new code corpus improves over the original DeepSeek-Coder corpus.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in the paper header: https://github.com/deepseek-ai/DeepSeek-Coder-V2. Model weights released under permissive license."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The training data (6T tokens from GitHub, CommonCrawl) is not released. Standard evaluation benchmarks (HumanEval, MBPP, etc.) are public, but the proprietary training corpus is not available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions for the benchmark evaluations are provided in the paper. The paper mentions using 'identical scripts and environments' for baselines but does not provide these."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates (e.g., '90.2% on HumanEval') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims superiority over various models based solely on comparing raw numbers. No statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context throughout — e.g., ablation study shows improvement from 30.5% to 37.2% on HumanEval. All comparison tables show both the proposed model and baselines' raw scores."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark sizes or number of evaluation examples. Standard benchmarks are used (HumanEval has 164 problems) but no discussion of whether these are sufficient for the claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be single-run greedy decoding (the paper states 'greedy search strategy'). The one exception is AIME where maj@64 is mentioned but this is a different metric, not variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baseline comparisons including CodeLlama, StarCoder, StarCoder2, DeepSeek-Coder, Codestral, Llama3, GPT-4 variants, Claude 3 Opus, and Gemini 1.5 Pro (Tables 3-10)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4o-0513, Claude 3 Opus, Gemini 1.5 Pro, Llama3-70B, and Codestral — all contemporary at time of publication (June 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides an ablation study comparing the new code corpus vs. old DeepSeek-Coder corpus using a 1B parameter model. Figure 3 compares reward model signal vs. compiler signal for RL training."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple benchmarks and metrics used: HumanEval, MBPP+, LiveCodeBench, USACO, SWE-bench, Aider, Defects4J, CruxEval, RepoBench, FIM tasks, GSM8K, MATH, AIME, MMLU, BBH, Arena-Hard, MT-Bench, etc."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs. All evaluations are automated (test case pass/fail, exact match, GPT-4 as judge for Arena-Hard/MT-Bench). The paper claims code quality but relies entirely on automated metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard held-out benchmarks used. For RepoBench, the paper explicitly uses only December 2023 data to avoid overlap with training data (cutoff November 2023). LiveCodeBench uses problems from Dec 2023-June 2024."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-language breakdowns across 14 programming languages in Table 3, per-difficulty breakdowns for LiveCodeBench (Easy/Medium/Hard in Table 4), per-context-length breakdowns for RepoBench (Table 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The conclusion discusses failure modes: 'there is still a significant gap in instruction-following capabilities compared to current state-of-the-art models like GPT-4 Turbo. This gap leads to poor performance in complex scenarios and tasks such as those in SWEbench.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results reported: DeepSeek-Coder-V2-Lite drops on knowledge-intensive benchmarks like TriviaQA (59.5% vs 65.2% for DeepSeek-V2-Lite). Performance gap on CruxEval vs closed-source models acknowledged. Instability during training with exponential normalization noted."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'performance comparable to GPT4-Turbo in code-specific tasks' and this is supported by Tables 3-9 showing competitive or superior results on most benchmarks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are limited and backed by controlled experiments. The corpus ablation (Table 1) uses the same 1B model architecture with different corpora. The reward model vs. compiler signal comparison (Figure 3) is a controlled comparison. Claims like 'continued pre-training enhances coding capabilities' are supported by before/after comparisons on the same architecture."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Breaking the Barrier of Closed-Source Models in Code Intelligence' — a very broad framing. While results show competitive performance on specific benchmarks, the paper acknowledges 'a significant gap in instruction-following capabilities' and poor SWE-bench performance. The title overclaims relative to the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, improved performance could partly be due to benchmark contamination (training data includes GitHub repos that may contain benchmark solutions), but this is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames benchmark performance (HumanEval, MBPP pass@1) as 'code intelligence' without discussing what code intelligence actually entails or whether these benchmarks adequately measure it. The gap between 'pass@1 on function-level benchmarks' and 'code intelligence' is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions given for baselines: GPT-4-1106, GPT-4-Turbo-0409, GPT-4o-0513. Own model sizes (16B/236B with active params 2.4B/21B) clearly specified. Table 2 details training settings."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The HumanEval prompt template is provided in footnote 4. Math evaluation prompt is given in footnote 9 ('Please reason step by step, and put your final answer within \\boxed{}'). FIM format tokens shown in Section 3.1."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Training hyperparameters reported in Section 3.3: AdamW with β1=0.9, β2=0.95, weight decay 0.1, cosine decay with 2000 warmup steps. SFT: lr 5e-6, batch 1M tokens, 1B tokens total. YARN hyperparameters: scale 40, α=1, β=32. Evaluation uses greedy search."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The model is evaluated directly on benchmarks via standard prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides detailed data filtering rules: average line length <100, max line length <1000, >25% alphabetic chars, XML filtering, HTML visible text ratio >20%, JSON/YAML 50-5000 chars. Near-deduplication applied. Three-iteration CommonCrawl pipeline with fastText classifier described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The conclusion briefly mentions the instruction-following gap but this is more of a future work direction than a limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. No mention of evaluation methodology concerns, benchmark contamination risks, or limitations of the comparison methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper does not state what the results do NOT show or what settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw evaluation outputs are not available. Only aggregate scores are reported. Training data is not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes data collection in detail: GitHub repos before November 2023, CommonCrawl pipeline with fastText classification using seed corpus from StackOverflow/PyTorch docs/StackExchange, three iterations of data collection yielding 1,170B code tokens and 221B math tokens."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks and public code repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: filtering rules → near-deduplication → 821B code + 185B code-related text from GitHub; fastText seed corpus → iterative CommonCrawl collection → 70B code-related + 221B math-related tokens; plus 94B additional GitHub source code from two iterations. Final composition: 60% code, 10% math, 30% NL."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section. DeepSeek-AI is the sole affiliation but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed under 'DeepSeek-AI'. The affiliation is clear."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "DeepSeek-AI is both the developer and evaluator of the model. The company has a direct commercial interest in the model performing well on benchmarks."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Authors presumably have financial interest in DeepSeek-AI's products but this is not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training data cutoff stated as 'before November 2023' for GitHub data (Section 2). The paper uses this to justify temporal splits for RepoBench evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "For RepoBench, the paper explicitly addresses overlap: 'we only use data from December 2023' to avoid data present in pre-training. For LiveCodeBench, the paper uses the subset from Dec 2023 to June 2024 since 'the cut-off of the training data is before November 2023.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While temporal splits are used for RepoBench and LiveCodeBench, the paper does not address contamination for older benchmarks like HumanEval (2021), MBPP (2021), GSM8K (2021), or MATH (2021), all of which were available online well before the November 2023 training cutoff. These are among the paper's most prominent results."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency reported despite the model being 236B parameters. Active parameter counts (2.4B, 21B) are given but no wall-clock time or API cost data."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Total training compute is not quantified in GPU hours or cost. The paper states 10.2T tokens were used for training but does not report the hardware or time required."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results appear to be single-run evaluations using greedy decoding."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Number of runs not stated for any benchmark evaluation. Greedy decoding implies single-run but this is not explicitly confirmed for all benchmarks."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported for training or evaluation. Learning rates and other hyperparameters are stated but not how they were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states hyperparameters are 'consistent with DeepSeek V2 methodology' but does not justify the selection or explain if any search was performed for the new training phase."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "DeepSeek-AI is evaluating its own model against competitors. The paper claims to use 'identical scripts and environments' for baselines but does not acknowledge the inherent bias of self-evaluation or discuss whether independent evaluation was performed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance vs. compute comparison. The 236B model uses 21B active params vs. competitors' full dense models but inference FLOPs are not compared. The paper notes the smaller active params as an advantage without quantifying."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses many benchmarks but does not discuss whether they actually measure 'code intelligence' as claimed. No discussion of construct validity for any benchmark."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used in the evaluations. Models are evaluated directly via standard prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage addressed for RepoBench (uses only December 2023 data) and LiveCodeBench (uses problems from Dec 2023-June 2024, after training cutoff). However, not addressed for older benchmarks."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage in the evaluation setup."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data and benchmark problems share structural similarities (e.g., HumanEval problems sourced from common programming patterns that appear in GitHub training data)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method applied. Temporal splits are used for two benchmarks but no membership inference, canary strings, or n-gram overlap analysis performed."
       }
     }
   },
   "claims": [
     {
       "claim": "DeepSeek-Coder-V2 achieves 90.2% on HumanEval, outperforming GPT-4 Turbo (88.2%) and rivaling GPT-4o (91.0%)",
       "evidence": "Table 3 shows Python HumanEval scores. Greedy decoding used for all models with identical scripts and environments.",
       "supported": "moderate"
     },
     {
       "claim": "DeepSeek-Coder-V2 achieves 75.7% on MATH, nearly matching GPT-4o's 76.6%",
       "evidence": "Table 9, zero-shot chain-of-thought evaluation as described in footnote 9.",
       "supported": "moderate"
     },
     {
       "claim": "The new code corpus improves over the DeepSeek-Coder corpus by 6.7% on HumanEval and 9.4% on MBPP",
       "evidence": "Table 1 ablation study with 1B parameter model: old corpus 30.5%/44.6% vs new corpus 37.2%/54.0% at 2T tokens.",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek-Coder-V2 is the first open-source model to surpass 10% on SWE-Bench",
       "evidence": "Table 7 shows 12.7% on SWE-Bench. Other open-source models score 0.0% (DS-Coder-Instruct, DS-Coder-V2-Lite) or 2.7% (Codestral).",
       "supported": "moderate"
     },
     {
       "claim": "Reward model signal outperforms raw compiler signal for RL training",
       "evidence": "Figure 3 shows reward model signal achieving higher Pass@1 on both LeetCode and LeetCode-zh compared to compiler signal and SFT baseline.",
       "supported": "moderate"
     },
     {
       "claim": "DeepSeek-Coder-V2 maintains comparable general language performance to DeepSeek-V2",
       "evidence": "Table 10 shows comparable or improved scores on BBH, MMLU, ARC, C-Eval, CMMLU. Some drops on TriviaQA (82.3 vs 86.7) and NaturalQuestions (47.5 vs 53.4).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "DeepSeek-AI evaluates DeepSeek-Coder-V2 against competitors. While baselines are reportedly run with identical scripts, there is inherent bias in a company benchmarking its own model — they have full control over prompt templates, evaluation harness details, and which benchmarks to report."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-point estimates with no error bars, confidence intervals, or multi-run variance. Given that benchmark scores can vary significantly across runs, prompts, and evaluation configurations, the reported differences may not be statistically meaningful."
     },
     {
       "flag": "Contamination risk for primary benchmarks",
       "detail": "HumanEval (2021), MBPP (2021), GSM8K (2021), and MATH (2021) were all published years before the November 2023 training cutoff. The paper addresses contamination only for RepoBench and LiveCodeBench but not for these older, headline benchmarks where contamination risk is highest."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity section. The conclusion's acknowledgment of instruction-following gaps reads more as future work than honest limitations analysis."
     },
     {
       "flag": "Overclaiming title",
       "detail": "'Breaking the Barrier of Closed-Source Models in Code Intelligence' is a broad claim. The model is competitive on specific benchmarks but falls significantly short on SWE-bench (12.7% vs 26.7% for GPT-4o) and instruction-following tasks, which are arguably more representative of real-world 'code intelligence.'"
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": [
         "M. Chen",
         "J. Tworek",
         "H. Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, foundational for code generation evaluation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": [
         "C. E. Jimenez",
         "J. Yang",
         "A. Wettig"
       ],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Real-world software engineering benchmark used to evaluate model capabilities."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": [
         "N. Jain",
         "K. Han",
         "A. Gu"
       ],
       "year": 2024,
       "relevance": "Contamination-free code benchmark using temporally-split competitive programming problems."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": [
         "J. Liu",
         "C. S. Xia",
         "Y. Wang",
         "L. Zhang"
       ],
       "year": 2023,
       "relevance": "EvalPlus/MBPP+ evaluation framework used in this paper for more rigorous code evaluation."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming – the rise of code intelligence",
       "authors": [
         "D. Guo",
         "Q. Zhu",
         "D. Yang"
       ],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Predecessor model; provides the base architecture and training methodology built upon in this work."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": [
         "B. Roziere",
         "J. Gehring",
         "F. Gloeckle"
       ],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code model baseline for comparison."
     },
     {
       "title": "StarCoder 2 and The Stack V2: The next generation",
       "authors": [
         "A. Lozhkov",
         "R. Li",
         "L. B. Allal"
       ],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Open-source code model and training data corpus baseline."
     },
     {
       "title": "RepoBench: Benchmarking repository-level code auto-completion systems",
       "authors": [
         "T. Liu",
         "C. Xu",
         "J. McAuley"
       ],
       "year": 2023,
       "relevance": "Repository-level code completion benchmark used to evaluate long-context code understanding."
     },
     {
       "title": "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models",
       "authors": [
         "Z. Shao",
         "P. Wang",
         "Q. Zhu"
       ],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Provides the math training data pipeline and GRPO RL algorithm reused in this work."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": [
         "J. Austin",
         "A. Odena",
         "M. Nye"
       ],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used for code generation evaluation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Open-source code model with 338 language support and 128K context that practitioners can deploy, though not as simple as an API call."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "An open-source model matching GPT-4 Turbo on code benchmarks was a notable achievement at the time, challenging the closed-source dominance narrative."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk themes are discussed in the paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The title explicitly frames it as 'breaking the barrier' of closed-source models, creating mild tension with OpenAI/Google/Anthropic, but doesn't directly accuse anyone."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Weights are publicly released on GitHub/HuggingFace under a permissive license, though running a 236B MoE model requires substantial hardware."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "DeepSeek gained significant recognition in the AI community and the paper directly benchmarks against GPT-4, Claude, and Gemini."
     }
   }
 }