ai-research-survey

scan-v5.json (29473B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Large Language Model Unlearning for Source Code",
     "authors": [
       "Xue Jiang",
       "Yihong Dong",
       "Huangzhao Zhang",
       "Tangxinyu Wang",
       "Zheng Fang",
       "Yingwei Ma",
       "Rongyu Cao",
       "Binhua Li",
       "Zhi Jin",
       "Wenpin Jiao",
       "Yongbin Li",
       "Ge Li"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2506.17125",
     "doi": "10.48550/arXiv.2506.17125"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims (superior PDR, robustness to adversarial attacks, broad LLM applicability) are directly supported by Table 1, Figures 3–5, and the multi-LLM experiments in Figure 4.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies (Figure 6) isolate the effects of loss function choice and hyperparameters; comparative experiments control for all variables except the unlearning method, providing adequate support for causal claims about PROD's advantages.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion state PROD 'holds significant implications for advancing reliable code generation' and 'consistently exhibiting improvements,' but experiments are limited to four 7B-parameter models; generalization to larger models or other architectures is not tested.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes PROD's success entirely to token-level granularity but does not consider alternative explanations such as whether the intentional memorization fine-tuning step inflates baseline degradation, or whether PROD's lower forget thoroughness on some metrics explains retained utility.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly labels 1−BLEU as 'an indicator of potential copyright infringement' and says model utility is 'estimated' through HumanEval, acknowledging the proxy nature of both measurements.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section; the conclusion lists only strengths and contributions without acknowledging scope constraints.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper; the intentional memorization fine-tuning step used for copyright/insecurity tasks (which may not reflect real-world memorization) is not flagged as a validity concern.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state what the results do NOT show; there is no explicit acknowledgment that findings are bounded to 7B models, specific task formats, or the tested code domains.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed: National Key R&D Program No. 2023YFB4503801, NSFC grants 62192733/62192730/62192731, and Major Program of Hubei Province No. 2023BAA024.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with Peking University and Tongyi Lab (Alibaba Group) are disclosed; the internship context is also noted.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The listed funders are Chinese government research agencies independent of the study outcome; however, two authors are affiliated with Alibaba's Tongyi Lab and one evaluated model (Qwen2.5-Coder-7B) is an Alibaba product, a potential unconflated interest.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided; the Alibaba affiliation of two authors evaluating an Alibaba model (Qwen2.5-Coder) is not flagged as a financial interest.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are precisely defined: 'unlearning' is formalized mathematically (Eq. 1), 'forget quality' and 'model utility' are defined with specific metrics, and 'PDR' is defined formally (Eq. 6).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four contributions are explicitly enumerated: (1) empirical investigation of existing methods on code tasks, (2) PROD method, (3) benchmark covering three tasks, and (4) PDR metric.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides detailed technical descriptions of all four baselines (GA, NPO, DPO, FLAT) including their formulations and explains why their coarse granularity fails on code, situating PROD as a targeted improvement.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Source code and data are released at https://github.com/jiangxxxue/PROD as stated in the abstract and contributions section.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All three forget datasets use publicly available corpora: The Stack (copyright), CyberSecEval/Purple Llama (insecure code), and VersiCode (deprecated APIs); HumanEval is used for utility.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions 4 NVIDIA A100 GPUs and Python-level hyperparameters but provides no requirements.txt, Dockerfile, or dependency specification in the paper text.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "While hyperparameters are listed in the implementation details section, no step-by-step reproduction instructions are included in the paper itself; readers must infer the pipeline from the description.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Main results in Table 1 report only point estimates for PDR; Figure 5 shows min/max ranges for adversarial attack results only, not for primary comparative claims.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to any comparative claims despite training being repeated five times with different seeds.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports a '124% average relative gain over the strongest competitor' in PDR, and absolute PDR values are given with baseline context in Table 1.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The choice of 100 files for the copyright forget set, 1,916 snippets for insecure code, and 252 packages for deprecated APIs is not justified by power analysis or statistical reasoning.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Five random seeds are used but Table 1 reports only mean PDR values without standard deviations; variance is only shown for adversarial attack results in Figure 5.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Four baselines are included: Gradient Ascent (GA), Direct Preference Optimization (DPO), Negative Preference Optimization (NPO), and FLAT.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All four baselines are from 2023–2025 publications representing the current state of LLM unlearning; FLAT (Wang et al. 2025) and NPO (Zhang et al. 2024) are very recent.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Figure 6 presents ablations on loss function type (CE, KL, JS), nucleus sampling threshold p (0.2, 0.4, 0.8, 0.9, 1.0), and suppression strength α across six values.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The evaluation uses task-specific forget quality metrics (1−BLEU, pass rate under static analysis, exact match), HumanEval pass rate for utility, and the composite PDR metric.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Three volunteer evaluators with 2+ years of software development experience conducted a blinded perceptual quality comparison of 20 samples per task; win rates are reported in Table 2.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Each task uses a held-out test structure: second half of copyright files for evaluation, separate test prompts for insecure code, and temporally split still-valid APIs for deprecated API evaluation.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports PDR scores separately for all three tasks (copyright, insecurity, deprecation), and Figure 3 shows separate scatter plots for each task.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Figure 1 presents a detailed case study of three failure modes in baselines (mute refusal, token collapse, syntactic incoherence); Figure 6 shows PROD's performance degradation under poor hyperparameter choices.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The ablation in Figure 6 explicitly shows that removing noise elimination (p=1.0) yields the worst performance, and that certain α values degrade forgetting or utility.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model versions are cited: CodeLlama-7B, Qwen2.5-Coder-7B, Deepseek-coder-6.7B, and Starcoder-7B, each referencing a specific technical report.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "This is a fine-tuning/unlearning paper, not a prompting study; the 'prompts' are the code prefixes from the benchmark datasets rather than engineered system instructions.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Learning rate (2e-5 for memorization, grid search from {1e-4 to 1e-6} for unlearning), batch size (32), epochs (10), optimizer (AdamW, weight_decay=0.01), sequence length (1024), and method-specific parameters (β=0.1, p=0.8, α=0) are all reported.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "This is not an agentic AI paper; no agentic scaffolding is used.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Preprocessing is documented for all tasks: copyright filtering criteria (removing test files, templates), insecure code adoption from CyberSecEval, and three-step deprecated API processing (version filtration, file filtration, temporal split).",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All source datasets are publicly available (The Stack, CyberSecEval, VersiCode, HumanEval), and the code repository is released.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data curation is described for each task with specific filtering criteria, dataset sizes (100 files, 1,916 snippets, 252 packages/3,449 snippets), and evaluation setup.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": true,
           "answer": true,
           "justification": "Human evaluator recruitment is minimally described: 'three independent volunteer evaluators (each with two years or more of software development experience).'",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from dataset curation through memorization fine-tuning, unlearning application, and evaluation is described in the Evaluation and Implementation Details sections.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The training data cutoffs for CodeLlama-7B, Qwen2.5-Coder-7B, Deepseek-coder-6.7B, and Starcoder-7B are not stated, which is relevant since the paper intentionally targets snippets these models may have memorized.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether HumanEval (used for model utility) was in the training data of the evaluated models, nor whether the Stack snippets used as forget data were actually in training corpora.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "HumanEval is a well-known benchmark likely present in all evaluated models' training data; this is not acknowledged as a potential confound for the utility measurement.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": true,
           "answer": false,
           "justification": "The perceptual quality study with human evaluators is not pre-registered.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": true,
           "answer": false,
           "justification": "No IRB or ethics approval is mentioned for the human evaluator study.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": true,
           "answer": false,
           "justification": "Only professional experience level (2+ years software development) is reported; no age, gender, or other demographic information is provided for the three evaluators.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": true,
           "answer": true,
           "justification": "The inclusion criterion '2+ years of software development experience' is stated for human evaluators.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": true,
           "answer": true,
           "justification": "'We randomly select 20 samples from each of the three downstream tasks' for human evaluation.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": true,
           "answer": true,
           "justification": "'Each human evaluator assesses all samples without knowing which approach produced each sample' — blinding is explicitly described.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "With only three evaluators in a one-shot perceptual study, attrition is not applicable.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference latency or cost is reported; the paper mentions hardware (4 A100 GPUs) but provides no inference timing data.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Hardware (4 NVIDIA A100 GPUs) is mentioned but total GPU-hours, training time, or compute cost is not stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Existing unlearning methods (GA, NPO, DPO, FLAT) cause near-total utility degradation when applied to source code at high forget quality.",
       "evidence": "Figure 3 shows all four baselines achieve ~90% forgetting but their HumanEval utility drops to near zero; Figure 1 illustrates mute refusal, token collapse, and syntactic incoherence.",
       "supported": "strong"
     },
     {
       "claim": "PROD achieves superior Pareto Dominance Ratio (PDR) over all baselines across all three tasks.",
       "evidence": "Table 1: PROD scores 41.8%, 70.4%, 58.0% PDR on copyright/insecurity/deprecation tasks vs best baselines of 31.5%, 24.5%, 51.6% respectively.",
       "supported": "strong"
     },
     {
       "claim": "PROD generalizes consistently across four distinct code LLMs of varying architectures.",
       "evidence": "Figure 4 shows all four models (CodeLlama-7B, Qwen2.5-Coder-7B, Deepseek-coder-6.7B, Starcoder-7B) achieve near-perfect forget quality while maintaining substantially higher utility than baselines.",
       "supported": "strong"
     },
     {
       "claim": "PROD is the only unlearning method robust to prefix injection adversarial attacks.",
       "evidence": "Figure 5 shows PROD's similarity to copyrighted code stays below 0.05 under attack while all baselines exceed 0.3; PROD also shows smallest variance.",
       "supported": "strong"
     },
     {
       "claim": "Token-level granularity (vs. sample-level) is the key mechanism preventing utility collapse.",
       "evidence": "The theoretical argument is coherent and supported by the contrast between PROD and baselines, but no direct ablation isolates granularity level as a variable independent of the redistribution mechanism.",
       "supported": "moderate"
     },
     {
       "claim": "PROD achieves >70% win rate against all baselines in human perceptual quality evaluation.",
       "evidence": "Table 2 shows win rates of 81%, 92%, 76%, 87% vs GA, DPO, NPO, FLAT respectively (human); GPT-4 evaluation shows 77%, 86%, 73%, 81%.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "PROD introduces token-level probabilistic redistribution for LLM code unlearning, surgically zeroing target token probabilities and renormalizing over remaining vocabulary to preserve programming language knowledge. In experiments across three unlearning tasks (copyright, insecure code, deprecated APIs) on four 7B code LLMs, PROD achieves 41–70% PDR versus near-zero for existing methods, which collapse model utility while achieving forget quality. PROD is also the only method resilient to prefix injection adversarial attacks, maintaining similarity below 0.05 to forgotten content. The paper simultaneously introduces a benchmark and PDR metric for multi-objective unlearning evaluation.",
   "red_flags": [
     {
       "flag": "Trivially small human study",
       "detail": "The perceptual quality evaluation uses only 3 volunteer evaluators assessing 20 samples per task — far too few for reliable inter-rater statistics or significance testing."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (PDR, win rates, adversarial attack robustness) are made without significance tests despite 5-seed training runs, making it impossible to assess whether differences are statistically meaningful."
     },
     {
       "flag": "Variance omitted from main results",
       "detail": "Table 1 reports only mean PDR values without standard deviations across the 5 random seeds; variance is reported only for adversarial attack results."
     },
     {
       "flag": "Artificial memorization step inflates baseline degradation",
       "detail": "For copyright and insecurity tasks, models are first fine-tuned on forget data to ensure memorization before unlearning. This setup may exaggerate how severely existing methods degrade utility compared to real-world memorization patterns."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated limitations or threats-to-validity section; scope boundaries (7B models only, specific task formats, English code) are not explicitly stated."
     },
     {
       "flag": "Alibaba affiliation evaluating Alibaba model",
       "detail": "Two authors are from Alibaba's Tongyi Lab and one of the four evaluated models is Qwen2.5-Coder-7B (an Alibaba product); no competing interest statement is provided."
     },
     {
       "flag": "HumanEval contamination unaddressed",
       "detail": "HumanEval is used as the sole proxy for model utility but is a well-known benchmark likely present in all evaluated models' training data; potential ceiling effects or contamination are not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Model Unlearning",
       "relevance": "Introduces the Gradient Ascent baseline and frames the LLM unlearning problem; primary foil for PROD."
     },
     {
       "title": "Negative Preference Optimization: From Catastrophic Collapse to Effective Unlearning",
       "relevance": "NPO baseline; DPO-derived unlearning method for LLMs."
     },
     {
       "title": "LLM Unlearning via Loss Adjustment with Only Forget Data (FLAT)",
       "relevance": "Most recent baseline; f-divergence based unlearning with template responses."
     },
     {
       "title": "TOFU: A Task of Fictitious Unlearning for LLMs",
       "relevance": "Standard unlearning benchmark for natural language; provides evaluation context."
     },
     {
       "title": "Rethinking machine unlearning for large language models",
       "relevance": "Survey/position paper on LLM unlearning landscape; provides problem framing."
     },
     {
       "title": "Safety Alignment Should be Made More Than Just a Few Tokens Deep",
       "relevance": "Source of prefix injection attack methodology used for adversarial robustness evaluation."
     },
     {
       "title": "VersiCode: Towards version-controllable code generation",
       "relevance": "Dataset used for deprecated API unlearning task; benchmark on version-aware code generation."
     },
     {
       "title": "Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models",
       "relevance": "Dataset used for insecure code unlearning task; 1,916 snippets with 50 CWE vulnerabilities."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses real-world pain points (copyright infringement, security vulnerabilities, deprecated APIs) in deployed code LLMs, with released code enabling adoption."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that state-of-the-art NLP unlearning methods catastrophically break code generation capability is genuinely surprising and challenges assumptions about method transferability."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Addresses copyright infringement legal risk and security vulnerability propagation from LLMs, both active concerns for AI deployment."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Some tension with existing methods (showing they all fail), but no major community controversy; technical disagreement rather than ideological conflict."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is released on GitHub and all datasets are publicly available, enabling practitioners to reproduce results on the three benchmark tasks."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Peking University and Alibaba Tongyi Lab are respectable institutions but not the most prominent names in the LLM unlearning community (DeepMind, OpenAI, CMU, etc.)."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45588752",
         "title": "Evaluating Argon2 adoption and effectiveness in real-world software",
         "points": 32,
         "comments": 32,
         "url": "https://news.ycombinator.com/item?id=45588752",
         "created_at": "2025-10-15T06:29:18Z"
       },
       {
         "hn_id": "40795525",
         "title": "Indications of superconductivities in blend of variant apatite and covellite",
         "points": 26,
         "comments": 20,
         "url": "https://news.ycombinator.com/item?id=40795525",
         "created_at": "2024-06-26T01:20:27Z"
       },
       {
         "hn_id": "43736950",
         "title": "Show HN: LettuceDetect – Lightweight hallucination detector for RAG pipelines",
         "points": 10,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43736950",
         "created_at": "2025-04-19T15:12:02Z"
       },
       {
         "hn_id": "45147728",
         "title": "Contemplative Artificial Intelligence",
         "points": 3,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=45147728",
         "created_at": "2025-09-06T09:02:21Z"
       },
       {
         "hn_id": "43740184",
         "title": "The Cambridge Report on Database Research",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43740184",
         "created_at": "2025-04-19T23:09:29Z"
       },
       {
         "hn_id": "36552652",
         "title": "GenAI for Programming Education: Benchmarking ChatGPT, GPT-4, and Human Tutors",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=36552652",
         "created_at": "2023-07-01T17:36:51Z"
       },
       {
         "hn_id": "42892730",
         "title": "Using Code Generation to Solve Open Instances of Combinatorial Design Problems",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42892730",
         "created_at": "2025-01-31T22:03:56Z"
       },
       {
         "hn_id": "41471269",
         "title": "Unifying Multimodal Retrieval via Document Screenshot Embedding",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41471269",
         "created_at": "2024-09-07T03:26:45Z"
       },
       {
         "hn_id": "45349444",
         "title": "Seeing Is Deceiving:Mirror-Based Lidar Spoofing for Autonomous Vehicle Deception",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45349444",
         "created_at": "2025-09-23T16:39:48Z"
       },
       {
         "hn_id": "44096948",
         "title": "Reasoning Model Is Stubborn: Instruction Overriding in Reasoning Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44096948",
         "created_at": "2025-05-26T12:44:46Z"
       }
     ],
     "top_points": 32,
     "total_points": 82,
     "total_comments": 56
   }
 }