scan-v5.json (30120B)
1 { 2 "scan_version": 5, 3 "paper_type": "empirical", 4 "paper": { 5 "title": "DeputyDev - AI Powered Developer Assistant: Breaking the Code Review Logjam through Contextual AI to Boost Developer Productivity", 6 "authors": [ 7 "Vishal Khare", 8 "V. Saini", 9 "Deepak Sharma", 10 "Anand Kumar", 11 "Ankit Rana" 12 ], 13 "year": 2025, 14 "venue": "arXiv.org", 15 "arxiv_id": "2508.09676", 16 "doi": "10.48550/arXiv.2508.09676" 17 }, 18 "checklist": { 19 "claims_and_evidence": { 20 "abstract_claims_supported": { 21 "applies": true, 22 "answer": false, 23 "justification": "Abstract claims 'statistically significant reduction' but paper reports only percentages (17-29%), no p-values or confidence intervals. Claims about rollout and SaaS availability are stated without evidence.", 24 "source": "haiku" 25 }, 26 "causal_claims_justified": { 27 "applies": true, 28 "answer": false, 29 "justification": "Paper claims DeputyDev 'causes' time savings but lacks statistical significance testing (no p-values). Aggressive filtering (removing size outliers, requiring balanced repos) creates selection bias. No alternative explanations discussed.", 30 "source": "haiku" 31 }, 32 "generalization_bounded": { 33 "applies": true, 34 "answer": false, 35 "justification": "Title ('Breaking the Code Review Logjam') makes broad claims but study evaluates only TATA 1mg, one organization, one version control system, 30-day window. No discussion of whether results apply to other teams, company sizes, or contexts.", 36 "source": "haiku" 37 }, 38 "alternative_explanations_discussed": { 39 "applies": true, 40 "answer": false, 41 "justification": "Paper explores only one explanation (DeputyDev helped). No discussion of: learning effects over time, reviewer quality variation, selection effects, or whether faster reviews correlate with code quality.", 42 "source": "haiku" 43 }, 44 "proxy_outcome_distinction": { 45 "applies": true, 46 "answer": false, 47 "justification": "Title claims 'developer productivity' improvement, but only measures 'review time' (hours). These are not equivalent—faster review could mean lower quality. Paper conflates them without justification.", 48 "source": "haiku" 49 } 50 }, 51 "limitations_and_scope": { 52 "limitations_section_present": { 53 "applies": true, 54 "answer": false, 55 "justification": "No dedicated limitations or threats-to-validity section. Conclusion only states findings are valuable, not what they fail to show. Filtering methodology is presented as design choice, not limitation.", 56 "source": "haiku" 57 }, 58 "threats_to_validity_specific": { 59 "applies": true, 60 "answer": false, 61 "justification": "Paper does not discuss specific threats like: 30-day window representativeness, selection bias from filtering, Hawthorne effect, reviewer expertise variation, or vendor lock-in to OpenAI/Anthropic.", 62 "source": "haiku" 63 }, 64 "scope_boundaries_stated": { 65 "applies": true, 66 "answer": false, 67 "justification": "Scope is implicit in experiment design but not explicitly bounded. No statement of what results do NOT show: generalizability to other orgs, long-term effects, code quality impact, or other LLM vendors.", 68 "source": "haiku" 69 } 70 }, 71 "conflicts_of_interest": { 72 "funding_disclosed": { 73 "applies": true, 74 "answer": false, 75 "justification": "No funding section. Paper neither discloses nor disclaims funding sources. Internal TATA 1mg research funding is implicit but unstated.", 76 "source": "haiku" 77 }, 78 "affiliations_disclosed": { 79 "applies": true, 80 "answer": true, 81 "justification": "All authors listed as 'TATA 1mg Healthcare Solutions Private Limited' employees. Affiliations are disclosed in author line.", 82 "source": "haiku" 83 }, 84 "funder_independent_of_outcome": { 85 "applies": true, 86 "answer": false, 87 "justification": "TATA 1mg (implicit funder) directly benefits from positive evaluation. Company develops and deploys DeputyDev as SaaS product. Funder is not independent of the outcome being evaluated.", 88 "source": "haiku" 89 }, 90 "financial_interests_declared": { 91 "applies": true, 92 "answer": false, 93 "justification": "No competing interests statement. Authors are full-time employees of the company commercializing DeputyDev as a SaaS product, creating direct financial interest in positive findings.", 94 "source": "haiku" 95 } 96 }, 97 "scope_and_framing": { 98 "key_terms_defined": { 99 "applies": true, 100 "answer": false, 101 "justification": "Key term 'productivity' used in title but never formally defined. Authors measure 'review time' as proxy without justifying equivalence. 'Statistically significant' claimed without p-values. 'Contextual AI' described only through implementation.", 102 "source": "haiku" 103 }, 104 "intended_contribution_clear": { 105 "applies": true, 106 "answer": false, 107 "justification": "Paper does not explicitly state its research question or contribution. Unclear whether it's a tool paper, empirical finding, or methodological advance. Title is a business claim ('breaking the logjam') not a research statement.", 108 "source": "haiku" 109 }, 110 "engagement_with_prior_work": { 111 "applies": true, 112 "answer": false, 113 "justification": "Related work cited (Tufano et al., Hong et al.) but not engaged with. No comparison of approach to prior code review automation. Section 6.2 quotes Andrew Ng but doesn't position against other agentic work. No related-work section.", 114 "source": "haiku" 115 } 116 } 117 }, 118 "type_checklist": { 119 "empirical": { 120 "artifacts": { 121 "code_released": { 122 "applies": true, 123 "answer": false, 124 "justification": "No code release. DeputyDev is a proprietary SaaS product. No GitHub, artifact repository, or release plan mentioned.", 125 "source": "haiku" 126 }, 127 "data_released": { 128 "applies": true, 129 "answer": false, 130 "justification": "Experimental data (721 PRs, review times) not released. No dataset URL or availability statement. Tables 2-3 show aggregated results only.", 131 "source": "haiku" 132 }, 133 "environment_specified": { 134 "applies": true, 135 "answer": false, 136 "justification": "Models named (GPT-4o, Claude 3.5 Sonnet) but no exact versions/snapshots. Integrations mentioned (Bitbucket, Jira, Confluence) but no deployment specs, requirements.txt, Dockerfile, or setup instructions.", 137 "source": "haiku" 138 }, 139 "reproduction_instructions": { 140 "applies": true, 141 "answer": false, 142 "justification": "No instructions for reproducing the evaluation or deploying DeputyDev. Appendix C shows mean/median formulas (standard definitions), not analysis steps. Cannot reproduce from this paper.", 143 "source": "haiku" 144 } 145 }, 146 "statistical_methodology": { 147 "confidence_intervals_or_error_bars": { 148 "applies": true, 149 "answer": false, 150 "justification": "Table 2 reports point estimates only: avg review times, per-LOC times, medians. Zero confidence intervals, error bars, or variance bounds reported.", 151 "source": "haiku" 152 }, 153 "significance_tests": { 154 "applies": true, 155 "answer": false, 156 "justification": "Paper claims 'statistically significant reduction' multiple times but reports zero p-values, t-tests, or hypothesis tests. Test vs control groups compared via percentage difference only.", 157 "source": "haiku" 158 }, 159 "effect_sizes_reported": { 160 "applies": true, 161 "answer": false, 162 "justification": "Percentage improvements reported (28.82%, 42.19%, etc.) but no formal effect sizes (Cohen's d, Hedges' g). Baseline context missing for some metrics (e.g., median review time baseline not clearly stated).", 163 "source": "haiku" 164 }, 165 "sample_size_justified": { 166 "applies": true, 167 "answer": false, 168 "justification": "No power analysis. Paper mentions 'over 200 engineers' but analyzes 721 PRs. Why 30 days? Why these sample sizes? No justification provided.", 169 "source": "haiku" 170 }, 171 "variance_reported": { 172 "applies": true, 173 "answer": false, 174 "justification": "Table 2 reports means and medians but no standard deviations, ranges, or quantiles. Figure 4 shows distributions visually but no numeric variance reported.", 175 "source": "haiku" 176 } 177 }, 178 "evaluation_design": { 179 "baselines_included": { 180 "applies": true, 181 "answer": true, 182 "justification": "Paper includes two control groups (CS1, CS2) without treatment. Test set compared to both controls. However, no comparison to other code review tools or LLM-based systems.", 183 "source": "haiku" 184 }, 185 "baselines_contemporary": { 186 "applies": true, 187 "answer": false, 188 "justification": "No external baselines compared (other code review tools, other LLM approaches). Only internal control groups. Therefore N/A for 'contemporary'.", 189 "source": "haiku" 190 }, 191 "ablation_study": { 192 "applies": true, 193 "answer": false, 194 "justification": "DeputyDev uses 6 agents, semantic search, AST chunking, reflection, blending engine. No ablation showing which components contribute to improvements. Cannot isolate component effects.", 195 "source": "haiku" 196 }, 197 "multiple_metrics": { 198 "applies": true, 199 "answer": true, 200 "justification": "Multiple metrics reported: avg review time/PR, avg time/LOC, median time, breakdown by PR size. Though correlated, these provide multiple lenses on the outcome.", 201 "source": "haiku" 202 }, 203 "human_evaluation": { 204 "applies": true, 205 "answer": false, 206 "justification": "No human raters evaluated AI-generated review quality. Appendix B shows example reviews but no systematic quality assessment. Only machine metric (review time) evaluated.", 207 "source": "haiku" 208 }, 209 "held_out_test_set": { 210 "applies": true, 211 "answer": false, 212 "justification": "Experiment uses concurrent A/B testing (test set concurrent with controls over 30 days) not a train/test split. Not a traditional held-out evaluation.", 213 "source": "haiku" 214 }, 215 "per_category_breakdown": { 216 "applies": true, 217 "answer": true, 218 "justification": "Table 3 breaks down results by PR size category (Small 0-50 LOC, Medium 51-100, Large 101-200, XL 201-500). Shows differential effects by category.", 219 "source": "haiku" 220 }, 221 "failure_cases_discussed": { 222 "applies": true, 223 "answer": false, 224 "justification": "Table 3 shows mixed/negative results (XL category shows 100.30% time INCREASE vs CS1) but no detailed analysis of why. Section 10.4 offers fixed-costs explanation but no specific failure examples.", 225 "source": "haiku" 226 }, 227 "negative_results_reported": { 228 "applies": true, 229 "answer": false, 230 "justification": "Table 3 shows concerning result for extra-large PRs (201-500 LOC): time actually increased 100.30% vs CS1. Presented without detailed discussion or implications.", 231 "source": "haiku" 232 } 233 }, 234 "setup_transparency": { 235 "model_versions_specified": { 236 "applies": true, 237 "answer": false, 238 "justification": "Model names given (GPT-4o, Claude 3.5 Sonnet) but no exact versions/snapshots. For example, Claude 3.5 Sonnet receives updates—which version? No commit hashes or release dates.", 239 "source": "haiku" 240 }, 241 "prompts_provided": { 242 "applies": true, 243 "answer": false, 244 "justification": "Appendix A shows XML output structure of agent responses, not the input prompts. Paper describes agent roles (Security, Code Communication, etc.) but actual prompts not provided.", 245 "source": "haiku" 246 }, 247 "hyperparameters_reported": { 248 "applies": true, 249 "answer": false, 250 "justification": "No temperature, top-p, max_tokens, or other LLM hyperparameters reported. No statement about defaults vs tuning.", 251 "source": "haiku" 252 }, 253 "scaffolding_described": { 254 "applies": true, 255 "answer": true, 256 "justification": "Agentic scaffolding well-documented: 6 agents (Security, Code Communication, Performance, Maintainability, Errors, Business Logic), multi-agent pattern, reflection pattern, blending engine with dimensions. Section 6.5 includes mathematical formulation.", 257 "source": "haiku" 258 }, 259 "data_preprocessing_documented": { 260 "applies": true, 261 "answer": true, 262 "justification": "Preprocessing steps documented: AST creation, semantic chunking, lexical+semantic search union, repo filtering (≥10 PRs/set), PR size filtering (remove top 25%, bottom 10%). Context assembly from multiple sources described.", 263 "source": "haiku" 264 } 265 }, 266 "data_integrity": { 267 "raw_data_available": { 268 "applies": true, 269 "answer": false, 270 "justification": "Raw PR data (individual review times for 721 PRs) not available. Only aggregated tables (Table 2-3) and distribution plots provided. No appendix with raw values.", 271 "source": "haiku" 272 }, 273 "data_collection_described": { 274 "applies": true, 275 "answer": false, 276 "justification": "Collection via Bitbucket webhook, 30-day window stated. But allocation mechanism unclear: was treatment randomized per PR? Per developer? Per repo? No specification of how the 33% split was enforced.", 277 "source": "haiku" 278 }, 279 "recruitment_methods_described": { 280 "applies": false, 281 "answer": false, 282 "justification": "Not a human subjects study. This is observational data collection from normal engineering workflow. No recruitment of participants.", 283 "source": "haiku" 284 }, 285 "data_pipeline_documented": { 286 "applies": true, 287 "answer": false, 288 "justification": "Implementation pipeline (PR → context → agents → blend → results) is described. But analysis pipeline incomplete: how were 721 PRs selected from 30-day corpus? How were CS1 and CS2 created? Step-by-step process not documented.", 289 "source": "haiku" 290 } 291 }, 292 "contamination": { 293 "training_cutoff_stated": { 294 "applies": false, 295 "answer": false, 296 "justification": "Not evaluating LLM benchmark performance. This measures tool's effect on code review workflow, not LLM capability on benchmarks.", 297 "source": "haiku" 298 }, 299 "train_test_overlap_discussed": { 300 "applies": false, 301 "answer": false, 302 "justification": "Not applicable to this evaluation type.", 303 "source": "haiku" 304 }, 305 "benchmark_contamination_addressed": { 306 "applies": false, 307 "answer": false, 308 "justification": "Not applicable to this evaluation type.", 309 "source": "haiku" 310 } 311 }, 312 "human_studies": { 313 "pre_registered": { 314 "applies": false, 315 "answer": false, 316 "justification": "No human subjects (observational PR data only). Pre-registration N/A.", 317 "source": "haiku" 318 }, 319 "irb_or_ethics_approval": { 320 "applies": false, 321 "answer": false, 322 "justification": "No human subjects involved. Company telemetry on its own engineers' workflow (likely covered by internal policies, not published).", 323 "source": "haiku" 324 }, 325 "demographics_reported": { 326 "applies": false, 327 "answer": false, 328 "justification": "N/A. No human subject demographics to report.", 329 "source": "haiku" 330 }, 331 "inclusion_exclusion_criteria": { 332 "applies": false, 333 "answer": false, 334 "justification": "N/A. No human subjects.", 335 "source": "haiku" 336 }, 337 "randomization_described": { 338 "applies": false, 339 "answer": false, 340 "justification": "N/A. No human subjects (though PR assignment to test/control groups may have been randomized—not explicitly stated).", 341 "source": "haiku" 342 }, 343 "blinding_described": { 344 "applies": false, 345 "answer": false, 346 "justification": "N/A. No human subjects.", 347 "source": "haiku" 348 }, 349 "attrition_reported": { 350 "applies": false, 351 "answer": false, 352 "justification": "N/A. No human subjects.", 353 "source": "haiku" 354 } 355 }, 356 "cost_and_practicality": { 357 "inference_cost_reported": { 358 "applies": true, 359 "answer": false, 360 "justification": "No inference cost data provided. Paper mentions cost as a reason to not use entire codebase (section 5) but reports zero actual costs or cost estimates.", 361 "source": "haiku" 362 }, 363 "compute_budget_stated": { 364 "applies": true, 365 "answer": false, 366 "justification": "No total computational budget, API costs, or infrastructure costs disclosed. No statement of cost per PR review or monthly operational cost.", 367 "source": "haiku" 368 } 369 } 370 } 371 }, 372 "claims": [ 373 { 374 "claim": "DeputyDev reduces code review time by 17-29% in average per-PR time and 38-42% in per-LOC time", 375 "evidence": "Table 2 shows test set 17.36% reduction vs CS1 and 28.82% vs CS2 in avg time; 42.19% and 38.98% reduction in time/LOC", 376 "supported": "moderate" 377 }, 378 { 379 "claim": "DeputyDev is most effective for small pull requests (0-50 LOC)", 380 "evidence": "Table 3 shows 43.87% reduction in per-LOC time for S category vs diminishing returns for larger PRs", 381 "supported": "moderate" 382 }, 383 { 384 "claim": "Code review time is weakly correlated with lines of code changed (r=0.004-0.095)", 385 "evidence": "Figure 5 shows correlation coefficients across all three sets, contradicting intuition", 386 "supported": "strong" 387 }, 388 { 389 "claim": "Multi-agent agentic workflow with reflection improves code review quality", 390 "evidence": "Appendix B shows example reviews; paper claims agents handle 6 aspects (security, communication, performance, maintainability, errors, business logic)", 391 "supported": "weak" 392 }, 393 { 394 "claim": "DeputyDev improves developer productivity by reducing context-switching delays", 395 "evidence": "Abstract and intro claim productivity gains, citing 23-min interruption cost from prior work. Measured via review-time reduction.", 396 "supported": "weak" 397 }, 398 { 399 "claim": "Contextual code understanding via AST parsing and semantic search produces better reviews", 400 "evidence": "Section 5-6 describes context assembly, section 6.1 details lexical+semantic search union. No ablation or comparison.", 401 "supported": "weak" 402 }, 403 { 404 "claim": "DeputyDev reduces median code review time by 46-47%", 405 "evidence": "Table 2 shows median reduction from 0.76-0.78 hours to 0.41 hours (47-46% decrease)", 406 "supported": "strong" 407 } 408 ], 409 "methodology_tags": [ 410 "observational", 411 "case-study", 412 "a-b-test" 413 ], 414 "key_findings": "DeputyDev, a multi-agent LLM-based code review tool, reduced average review time by 17–28% and median review time by 46–47% in a 30-day A/B test at TATA 1mg covering 721 pull requests. Effectiveness is inversely proportional to PR size: 43.87% per-LOC time reduction for small PRs (0–50 LOC) versus mixed results for extra-large PRs (201–500 LOC). The analysis found a weak correlation (0.004–0.095) between code volume and review duration, suggesting complexity rather than quantity drives review time. However, the study lacks statistical significance testing, human evaluation of review quality, and external validation.", 415 "red_flags": [ 416 { 417 "flag": "No statistical significance testing", 418 "detail": "Paper repeatedly claims 'statistically significant' results but reports zero p-values, confidence intervals, or hypothesis tests. Only percentage differences shown." 419 }, 420 { 421 "flag": "Self-interested internal evaluation", 422 "detail": "All authors are TATA 1mg employees evaluating TATA 1mg's product. No independent third-party validation. Company commercializes tool as SaaS, creating direct financial interest in positive findings." 423 }, 424 { 425 "flag": "No conflicts of interest statement", 426 "detail": "Missing explicit declaration that authors benefit from positive evaluation outcomes and control the evaluation methodology." 427 }, 428 { 429 "flag": "Aggressive filtering creates selection bias", 430 "detail": "Removed top 25% and bottom 10% of PRs by size, required balanced repositories. Creates a selected subset that may not be representative of real-world code review." 431 }, 432 { 433 "flag": "No code quality evaluation", 434 "detail": "Only measured review time (machine metric). Zero human raters assessing whether AI-generated reviews are actually helpful or correct. Appendix B shows examples only." 435 }, 436 { 437 "flag": "Productivity claim unsubstantiated", 438 "detail": "Title claims 'boost developer productivity' but only measures 'review time'. These are not equivalent—faster review could signal lower quality. No evidence for productivity claim." 439 }, 440 { 441 "flag": "No ablation study", 442 "detail": "System uses semantic search, AST chunking, 6 agents, reflection, blending engine. Cannot isolate which components actually contribute to improvements." 443 }, 444 { 445 "flag": "Prompts and LLM versions not disclosed", 446 "detail": "Model names provided (GPT-4o, Claude 3.5 Sonnet) but no exact versions/snapshots. Actual prompts fed to LLM agents not provided—critical for reproducibility." 447 }, 448 { 449 "flag": "Confounded control groups", 450 "detail": "Paper defines Control Set 1 and Control Set 2 but never explains the difference between them or why two controls are needed. Unclear what is actually being measured." 451 }, 452 { 453 "flag": "Short evaluation window without long-term data", 454 "detail": "Only 30 days of data (July 27 – Aug 27, 2024). No discussion of whether time savings persist, whether reviewers' behavior stabilizes, or if effects decay over time." 455 }, 456 { 457 "flag": "Single-organization study with no generalization evidence", 458 "detail": "Evaluated only at TATA 1mg on Bitbucket. No evidence this works for other companies, team sizes, code languages, or VCS platforms. Title overgeneralizes ('Breaking the Code Review Logjam')." 459 }, 460 { 461 "flag": "Concerning result for large PRs hidden in table", 462 "detail": "Table 3 shows extra-large PRs (201-500 LOC) had 100.30% TIME INCREASE in test vs CS1—opposite of claimed benefit. Presented without analysis or discussion of implications." 463 }, 464 { 465 "flag": "No data or code release for reproducibility", 466 "detail": "Raw PR data, code, and trained models not released. System is proprietary SaaS. Impossible for others to reproduce or validate findings." 467 } 468 ], 469 "cited_papers": [ 470 { 471 "title": "Code Time Report", 472 "authors": "software.com", 473 "year": 2024, 474 "relevance": "Motivation for study: cites 41 minutes/day on code review. Used to establish problem scope." 475 }, 476 { 477 "title": "CommentFinder: A Simpler, Faster, More Accurate Code Review Comments Recommendation", 478 "authors": "Hong et al.", 479 "year": 2022, 480 "relevance": "Prior automated code review work using NLP. Directly relevant comparison point." 481 }, 482 { 483 "title": "Code Review Automation: Strengths and Weaknesses of the State of the Art", 484 "authors": "Tufano, Dabić, Mastropaolo, Ciniselli, Bavota", 485 "year": 2024, 486 "relevance": "Systematic review of code review automation. Primary prior work in automated code review." 487 }, 488 { 489 "title": "Using Pre-trained Models to Boost Code Review Automation", 490 "authors": "Tufano, Masiero, Mastropaolo, Pascarella, Poshyvanyk, Bavota", 491 "year": 2022, 492 "relevance": "Earlier work on LLM-based code review automation. Methodological precedent." 493 }, 494 { 495 "title": "Agentic Design Patterns Part 5: Multi-Agent Collaboration", 496 "authors": "Andrew Ng", 497 "year": 2024, 498 "relevance": "Framework for multi-agent LLM systems. Theoretical foundation for DeputyDev's architecture." 499 }, 500 { 501 "title": "ChatDev: Communicative Agents for Software Development", 502 "authors": "Chen Qian, Wei Liu, Hongzhang Liu, et al.", 503 "year": 2024, 504 "relevance": "Multi-agent framework for software engineering tasks. Related agentic approach." 505 }, 506 { 507 "title": "The Cost of Interrupted Work: More Speed and Stress", 508 "authors": "Mark, Gudith, Klocke", 509 "year": 2008, 510 "relevance": "Foundational motivation: interruptions cause 23-minute context-switch cost. Cited as justification for review-time reduction mattering." 511 }, 512 { 513 "title": "Self-Refine: Iterative Refinement with Self-Feedback", 514 "authors": "Madaan, Tandon, Gupta, et al.", 515 "year": 2023, 516 "relevance": "Reflection pattern in LLMs. Methodological component of DeputyDev's approach." 517 }, 518 { 519 "title": "Introducing Structured Outputs in the API", 520 "authors": "OpenAI", 521 "year": 2024, 522 "relevance": "Technical capability for enforcing JSON output from LLMs. Implementation detail discussed in section 6.3." 523 } 524 ], 525 "engagement_factors": { 526 "practical_relevance": { 527 "score": 2, 528 "justification": "Code review assistant has clear practical value for development teams, but results limited to one company; practitioners cannot generalize to their context." 529 }, 530 "surprise_contrarian": { 531 "score": 1, 532 "justification": "Finding that AI assists code review is unsurprising. Weak LOC-time correlation is mildly interesting but cited as known phenomenon; no novel insights." 533 }, 534 "fear_safety": { 535 "score": 0, 536 "justification": "No AI safety or alignment concerns raised. Tool is narrow code review assistant, not generally capable system. No risk discussion." 537 }, 538 "drama_conflict": { 539 "score": 1, 540 "justification": "Minimal conflict: company employees evaluating their own product with no independent verification. Potential for bias is real but not explored or disputed." 541 }, 542 "demo_ability": { 543 "score": 2, 544 "justification": "Available as SaaS product (in principle), but paper does not explain how to access it, try it, or what it costs. Requires Bitbucket VCS." 545 }, 546 "brand_recognition": { 547 "score": 1, 548 "justification": "TATA 1mg is major Indian healthcare company but not prominent in AI research. Authors are not well-known researchers. No prestigious institution affiliation." 549 } 550 }, 551 "hn_data": { 552 "threads": [ 553 { 554 "hn_id": "36965545", 555 "title": "Electronic Structure of LK-99", 556 "points": 551, 557 "comments": 432, 558 "url": "https://news.ycombinator.com/item?id=36965545" 559 }, 560 { 561 "hn_id": "44016621", 562 "title": "LLMs are more persuasive than incentivized human persuaders", 563 "points": 140, 564 "comments": 116, 565 "url": "https://news.ycombinator.com/item?id=44016621" 566 }, 567 { 568 "hn_id": "43075571", 569 "title": "ZeroBench: An Impossible Visual Benchmark for Contemporary LMMs", 570 "points": 9, 571 "comments": 3, 572 "url": "https://news.ycombinator.com/item?id=43075571" 573 }, 574 { 575 "hn_id": "44211052", 576 "title": "Analog Foundation Models", 577 "points": 8, 578 "comments": 1, 579 "url": "https://news.ycombinator.com/item?id=44211052" 580 }, 581 { 582 "hn_id": "44009574", 583 "title": "Large Language Models Are More Persuasive Than Incentivized Human Persuaders", 584 "points": 4, 585 "comments": 1, 586 "url": "https://news.ycombinator.com/item?id=44009574" 587 }, 588 { 589 "hn_id": "45241249", 590 "title": "The Illusion of Diminishing Returns: Measuring Long Horizon Execution in LLMs", 591 "points": 4, 592 "comments": 0, 593 "url": "https://news.ycombinator.com/item?id=45241249" 594 }, 595 { 596 "hn_id": "45240847", 597 "title": "ButterflyQuant: Ultra-low-bit LLM Quantization", 598 "points": 4, 599 "comments": 0, 600 "url": "https://news.ycombinator.com/item?id=45240847" 601 }, 602 { 603 "hn_id": "45228682", 604 "title": "The Illusion of Diminishing Returns: Measuring Long Horizon Execution in LLMs", 605 "points": 3, 606 "comments": 1, 607 "url": "https://news.ycombinator.com/item?id=45228682" 608 }, 609 { 610 "hn_id": "45343343", 611 "title": "The illusion of diminishing returns in LLM progress", 612 "points": 3, 613 "comments": 0, 614 "url": "https://news.ycombinator.com/item?id=45343343" 615 }, 616 { 617 "hn_id": "43905563", 618 "title": "(How) Do reasoning models reason?", 619 "points": 3, 620 "comments": 0, 621 "url": "https://news.ycombinator.com/item?id=43905563" 622 } 623 ], 624 "top_points": 551, 625 "total_points": 729, 626 "total_comments": 554 627 } 628 }