scan-v5.json (26068B)
1 { 2 "scan_version": 5, 3 "paper_type": "empirical", 4 "paper": { 5 "title": "GLAD: Neural Predicate Synthesis to Repair Omission Faults", 6 "authors": [ 7 "Sungmin Kang", 8 "Shin Yoo" 9 ], 10 "year": 2022, 11 "venue": "2023 IEEE/ACM 45th International Conference on Software Engineering: Companion Proceedings (ICSE-Companion)", 12 "arxiv_id": "2204.06771", 13 "doi": "10.1109/ICSE-Companion58688.2023.00087" 14 }, 15 "checklist": { 16 "claims_and_evidence": { 17 "abstract_claims_supported": { 18 "applies": true, 19 "answer": true, 20 "justification": "The abstract claim that GLAD fixes 16 Defects4J v1.2 faults no previous NMT-based technique could is directly supported by Table 3 (DL column), and the 'reasonable runtime cost' claim is supported by Table 7.", 21 "source": "haiku" 22 }, 23 "causal_claims_justified": { 24 "applies": true, 25 "answer": true, 26 "justification": "RQ3 performs a dedicated ablation study removing each component (-Finetuning, -Grammar, -Language Model) and measures the performance impact in Table 5, providing adequate causal evidence for each component's contribution.", 27 "source": "haiku" 28 }, 29 "generalization_bounded": { 30 "applies": true, 31 "answer": true, 32 "justification": "The paper explicitly bounds generalization: 'further research is necessary to understand how far the performance of GLAD generalizes' and 'the performance of GLAD under such conditions [other programming languages] is currently unknown' (Section 7).", 33 "source": "haiku" 34 }, 35 "alternative_explanations_discussed": { 36 "applies": true, 37 "answer": false, 38 "justification": "The paper does not discuss alternative explanations for GLAD's superiority, such as whether the larger beam width (10,000 vs. prior work) or the restricted scope to if-statement faults inflates comparisons; only the ablation design addresses confounds.", 39 "source": "haiku" 40 }, 41 "proxy_outcome_distinction": { 42 "applies": true, 43 "answer": true, 44 "justification": "The paper clearly distinguishes 'plausible' patches (pass all tests) from 'correct' patches (manually verified as semantically equivalent to the developer patch), explicitly using both metrics throughout.", 45 "source": "haiku" 46 } 47 }, 48 "limitations_and_scope": { 49 "limitations_section_present": { 50 "applies": true, 51 "answer": true, 52 "justification": "Section 7 is titled 'THREATS TO VALIDITY' and addresses both internal and external validity in dedicated paragraphs, exceeding a single sentence.", 53 "source": "haiku" 54 }, 55 "threats_to_validity_specific": { 56 "applies": true, 57 "answer": true, 58 "justification": "The threats section names specific concerns: comparison against 40 tools to address internal validity, Java-only limitation for external validity, and explicit note that Defects4J v2.0 comparisons use different FL settings making them non-comparable.", 59 "source": "haiku" 60 }, 61 "scope_boundaries_stated": { 62 "applies": true, 63 "answer": true, 64 "justification": "The paper explicitly states GLAD targets only 'if-statement omission faults' in Java, and states that performance on other programming languages 'is currently unknown', bounding the scope of the claims.", 65 "source": "haiku" 66 } 67 }, 68 "conflicts_of_interest": { 69 "funding_disclosed": { 70 "applies": true, 71 "answer": false, 72 "justification": "No funding sources are mentioned anywhere in the paper.", 73 "source": "haiku" 74 }, 75 "affiliations_disclosed": { 76 "applies": true, 77 "answer": true, 78 "justification": "Both authors list KAIST (Korea Advanced Institute of Science and Technology) as their affiliation in the paper header.", 79 "source": "haiku" 80 }, 81 "funder_independent_of_outcome": { 82 "applies": false, 83 "answer": false, 84 "justification": "No funding is disclosed, so this criterion is not applicable.", 85 "source": "haiku" 86 }, 87 "financial_interests_declared": { 88 "applies": true, 89 "answer": false, 90 "justification": "There is no competing interests or financial disclosure statement anywhere in the paper.", 91 "source": "haiku" 92 } 93 }, 94 "scope_and_framing": { 95 "key_terms_defined": { 96 "applies": true, 97 "answer": true, 98 "justification": "Key terms are defined precisely: 'weak omission,' 'strong omission,' and 'if omission' faults are all defined in Section 3, and 'omission fault' is introduced in the introduction with a clear description.", 99 "source": "haiku" 100 }, 101 "intended_contribution_clear": { 102 "applies": true, 103 "answer": true, 104 "justification": "The paper provides a bulleted list of four explicit contributions in Section 1: the Defects4J analysis, the GLAD technique, its evaluation showing orthogonality, ablation studies, and public tool release.", 105 "source": "haiku" 106 }, 107 "engagement_with_prior_work": { 108 "applies": true, 109 "answer": true, 110 "justification": "Section 2 engages substantively with template-based techniques (TBar, FixMiner), NMT-based repair (SequenceR, CODIT, Recoder), condition synthesis (ACS, Nopol, Dynamoth), and language models—explicitly stating how GLAD differs from each category.", 111 "source": "haiku" 112 } 113 } 114 }, 115 "type_checklist": { 116 "empirical": { 117 "artifacts": { 118 "code_released": { 119 "applies": true, 120 "answer": true, 121 "justification": "The paper states 'we make our tool publicly available' and links to https://anonymous.4open.science/r/neural-pred-synth-4816/README.md; while this is an anonymized review link, the code and raw data (fix_data.csv) are accessible.", 122 "source": "haiku" 123 }, 124 "data_released": { 125 "applies": true, 126 "answer": true, 127 "justification": "The primary evaluation uses Defects4J (a standard public benchmark), and the authors also release their fix_data.csv via the same anonymous repository link.", 128 "source": "haiku" 129 }, 130 "environment_specified": { 131 "applies": true, 132 "answer": false, 133 "justification": "The paper specifies hardware (i7-6700, NVIDIA 1080/3090) and names libraries (javalang, JDB) but provides no requirements.txt, Dockerfile, or equivalent dependency specification.", 134 "source": "haiku" 135 }, 136 "reproduction_instructions": { 137 "applies": true, 138 "answer": false, 139 "justification": "The paper describes the algorithm in detail but provides no step-by-step instructions for running GLAD; readers are directed to the code repository without installation or execution guidance.", 140 "source": "haiku" 141 } 142 }, 143 "statistical_methodology": { 144 "confidence_intervals_or_error_bars": { 145 "applies": true, 146 "answer": false, 147 "justification": "All results are reported as raw counts of bugs fixed (Tables 3–7) with no confidence intervals or error bars reported for any main result.", 148 "source": "haiku" 149 }, 150 "significance_tests": { 151 "applies": true, 152 "answer": false, 153 "justification": "No statistical significance tests are performed; comparisons between GLAD and baselines are made purely by counting correctly fixed bugs.", 154 "source": "haiku" 155 }, 156 "effect_sizes_reported": { 157 "applies": true, 158 "answer": true, 159 "justification": "The paper reports absolute counts with clear baseline context (GLAD: 28, TBar: 9, all DL: 12 in Table 3), making effect magnitude directly interpretable.", 160 "source": "haiku" 161 }, 162 "sample_size_justified": { 163 "applies": true, 164 "answer": true, 165 "justification": "The paper evaluates on all 95 applicable bugs from Defects4J v1.2 and 89 from v2.0 (184 total if-statement faults), using the complete available benchmark rather than a sample.", 166 "source": "haiku" 167 }, 168 "variance_reported": { 169 "applies": true, 170 "answer": false, 171 "justification": "Table 7 reports median runtime but no variance, standard deviation, or range across multiple runs; deterministic repair means repeated-run variance is not discussed.", 172 "source": "haiku" 173 } 174 }, 175 "evaluation_design": { 176 "baselines_included": { 177 "applies": true, 178 "answer": true, 179 "justification": "Multiple baselines are used: TBar (template-based), 7+ learning-based DL tools (SequenceR, CURE, Recoder, etc.), and Restore for the method-given FL setting.", 180 "source": "haiku" 181 }, 182 "baselines_contemporary": { 183 "applies": true, 184 "answer": true, 185 "justification": "The primary baselines include Recoder (2021), CURE (2021), and Restore (recent), all published close in time to GLAD's 2022 submission.", 186 "source": "haiku" 187 }, 188 "ablation_study": { 189 "applies": true, 190 "answer": true, 191 "justification": "RQ3 presents a full ablation study (Table 5) removing finetuning, grammar, and language model independently, reporting plausible/correct counts and timing for each variant.", 192 "source": "haiku" 193 }, 194 "multiple_metrics": { 195 "applies": true, 196 "answer": true, 197 "justification": "The evaluation uses: number of plausible patches, correct patches, runtime (min/median/mean), MRR, acc@1, acc@5, and worst-case rank for the dynamic reranking analysis.", 198 "source": "haiku" 199 }, 200 "human_evaluation": { 201 "applies": true, 202 "answer": true, 203 "justification": "Plausible patches are 'manually inspected to check for correctness' and deemed correct only if judged 'semantically equivalent to the developer patch' (Section 4.4).", 204 "source": "haiku" 205 }, 206 "held_out_test_set": { 207 "applies": true, 208 "answer": true, 209 "justification": "Defects4J projects are explicitly purged from the LM pretraining dataset, ensuring the test bugs are held out from training data (Section 5.1.2).", 210 "source": "haiku" 211 }, 212 "per_category_breakdown": { 213 "applies": true, 214 "answer": true, 215 "justification": "Table 2 breaks down unfixed faults by omission type (weak, strong, if), Table 3 provides per-bug results across all 31 bugs, and Table 4 separates v1.2 from v2.0 results.", 216 "source": "haiku" 217 }, 218 "failure_cases_discussed": { 219 "applies": true, 220 "answer": true, 221 "justification": "RQ5 explicitly analyzes failure cases (Table 9: Math-106 and Time-27) with detailed explanations of why GLAD failed to produce the correct patch in each instance.", 222 "source": "haiku" 223 }, 224 "negative_results_reported": { 225 "applies": true, 226 "answer": true, 227 "justification": "The paper reports that GLAD generates the correct predicate for 32 bugs it ultimately fails to fix (due to body synthesis limitations), and documents the Defects4J v2.0 comparison caveats explicitly.", 228 "source": "haiku" 229 } 230 }, 231 "setup_transparency": { 232 "model_versions_specified": { 233 "applies": true, 234 "answer": true, 235 "justification": "The LM is a custom GRU trained on Java-med dataset; GRU architecture details are provided in the appendix, BPE with ~5,000 pairs is specified, and pretraining details (5 epochs) are given.", 236 "source": "haiku" 237 }, 238 "prompts_provided": { 239 "applies": true, 240 "answer": true, 241 "justification": "The 'repair seed' mechanism is fully described: an 'if' token plus opening parenthesis appended to the tokenized method context, with the special case for existing if-conditions also described (Section 4.1.2).", 242 "source": "haiku" 243 }, 244 "hyperparameters_reported": { 245 "applies": true, 246 "answer": true, 247 "justification": "Key hyperparameters are reported: beam width W=10,000, max token length l=15, finetuning for 1 epoch, pretraining for 5 epochs, BPE with ~5,000 pairs, dynamic reranking timeout of 15 minutes, and overall timeout of 3 hours.", 248 "source": "haiku" 249 }, 250 "scaffolding_described": { 251 "applies": true, 252 "answer": true, 253 "justification": "The full 6-step repair pipeline is described with a diagram (Figure 2) and Algorithm 1 pseudocode, covering finetuning, seed addition, grammar extraction, beam search, dynamic reranking, and patch validation.", 254 "source": "haiku" 255 }, 256 "data_preprocessing_documented": { 257 "applies": true, 258 "answer": true, 259 "justification": "Preprocessing is documented: javalang parsing, BPE subtokenization, purging Defects4J projects from pretraining, discarding <1% unparseable methods, and extraction of legal identifiers via AST analysis.", 260 "source": "haiku" 261 } 262 }, 263 "data_integrity": { 264 "raw_data_available": { 265 "applies": true, 266 "answer": true, 267 "justification": "The paper states 'the raw data used in this section is made publicly available' and links to fix_data.csv in the anonymous repository.", 268 "source": "haiku" 269 }, 270 "data_collection_described": { 271 "applies": true, 272 "answer": true, 273 "justification": "The collection procedure for APR results is described with explicit criteria: surveying program-repair.org and Monperrus' living APR review, with specific rules for which fault localization results to use (Section 3).", 274 "source": "haiku" 275 }, 276 "recruitment_methods_described": { 277 "applies": false, 278 "answer": false, 279 "justification": "This is a benchmark study using existing Defects4J bugs; no human participants are recruited.", 280 "source": "haiku" 281 }, 282 "data_pipeline_documented": { 283 "applies": true, 284 "answer": true, 285 "justification": "The pipeline from surveying APR tools → collecting per-bug fix results → feature extraction via git diff/Defects4J-dissection → statistical analysis is described in Section 3.", 286 "source": "haiku" 287 } 288 }, 289 "contamination": { 290 "training_cutoff_stated": { 291 "applies": true, 292 "answer": true, 293 "justification": "The paper states the LM is pretrained on the Java-med dataset with Defects4J projects explicitly purged prior to training (Section 5.1.2).", 294 "source": "haiku" 295 }, 296 "train_test_overlap_discussed": { 297 "applies": true, 298 "answer": true, 299 "justification": "Train-test overlap is directly addressed: 'Prior to training, data related to the Defects4J projects is purged' to prevent the LM from seeing the evaluation bugs during pretraining.", 300 "source": "haiku" 301 }, 302 "benchmark_contamination_addressed": { 303 "applies": true, 304 "answer": true, 305 "justification": "The paper explicitly purges Defects4J projects from the pretraining corpus, and evaluates generalization to Defects4J v2.0 bugs (which were introduced after the original v1.2 training data collection).", 306 "source": "haiku" 307 } 308 }, 309 "human_studies": { 310 "pre_registered": { 311 "applies": false, 312 "answer": false, 313 "justification": "No human participants; not applicable.", 314 "source": "haiku" 315 }, 316 "irb_or_ethics_approval": { 317 "applies": false, 318 "answer": false, 319 "justification": "No human participants; not applicable.", 320 "source": "haiku" 321 }, 322 "demographics_reported": { 323 "applies": false, 324 "answer": false, 325 "justification": "No human participants; not applicable.", 326 "source": "haiku" 327 }, 328 "inclusion_exclusion_criteria": { 329 "applies": false, 330 "answer": false, 331 "justification": "No human participants; not applicable.", 332 "source": "haiku" 333 }, 334 "randomization_described": { 335 "applies": false, 336 "answer": false, 337 "justification": "No human participants; not applicable.", 338 "source": "haiku" 339 }, 340 "blinding_described": { 341 "applies": false, 342 "answer": false, 343 "justification": "No human participants; not applicable.", 344 "source": "haiku" 345 }, 346 "attrition_reported": { 347 "applies": false, 348 "answer": false, 349 "justification": "No human participants; not applicable.", 350 "source": "haiku" 351 } 352 }, 353 "cost_and_practicality": { 354 "inference_cost_reported": { 355 "applies": true, 356 "answer": true, 357 "justification": "Table 7 reports minimum, median, and mean runtime to first plausible patch: median 9.38 minutes (perfect FL) and 18.59 minutes (method-given FL), with a 3-hour max timeout.", 358 "source": "haiku" 359 }, 360 "compute_budget_stated": { 361 "applies": true, 362 "answer": false, 363 "justification": "Hardware is specified (i7-6700, NVIDIA 1080/3090) but total GPU-hours used for pretraining or the full evaluation is not reported.", 364 "source": "haiku" 365 } 366 } 367 } 368 }, 369 "claims": [ 370 { 371 "claim": "GLAD fixes 16 Defects4J v1.2 faults that no previous learning-based (NMT-based) APR technique could fix.", 372 "evidence": "Table 3 directly lists per-bug results; 16 bugs have GLAD=correct and DL=not-fixed.", 373 "supported": "strong" 374 }, 375 { 376 "claim": "GLAD fixes 8 Defects4J faults that no APR tool among the 40 surveyed has ever fixed.", 377 "evidence": "Table 3, 'Ever' column shows 0 for 8 bugs (Cl-1, Cl-15, Cl-52, M-28, M-48, Mo-24, Mo-34, L-9) that GLAD fixes correctly.", 378 "supported": "strong" 379 }, 380 { 381 "claim": "Omission faults constitute 81.3% of unfixed Defects4J bugs in the 16–63 added-character range.", 382 "evidence": "Table 2 reports 39/48 manually inspected unfixed faults contain an omission aspect.", 383 "supported": "strong" 384 }, 385 { 386 "claim": "Each component (finetuning, grammar, language model) contributes independently to GLAD's repair performance.", 387 "evidence": "Ablation Table 5: removing grammar drops correct repairs from 48 to 21; removing LM drops to 1; removing finetuning drops to 39.", 388 "supported": "strong" 389 }, 390 { 391 "claim": "GLAD generates most correct patches in under 30 minutes, comparable to prior state-of-the-art runtime.", 392 "evidence": "Table 7: median perfect-FL time is 9.38 min vs. CURE's 16.5 min mean; more than half of method-given bugs fixed in under 20 minutes.", 393 "supported": "strong" 394 }, 395 { 396 "claim": "Dynamic reranking reduces required patch validation effort by orders of magnitude.", 397 "evidence": "Table 6: worst-case rank improves from 28,321 to 745 with dynamic reranking; acc@1 improves from 5 to 18.", 398 "supported": "strong" 399 } 400 ], 401 "methodology_tags": [ 402 "benchmark-eval", 403 "empirical" 404 ], 405 "key_findings": "GLAD is a learning-based APR technique combining generative language models, type-based grammar-constrained beam search, and debugger-based dynamic reranking to synthesize if-statement predicates for omission faults. On Defects4J v1.2, GLAD correctly fixes 28 bugs under perfect fault localization, including 16 never fixed by any prior learning-based technique and 8 never fixed by any of 40 APR tools surveyed. Ablation results show grammar and the language model are both essential: removing the grammar halves correct repairs, while removing the LM reduces them to nearly zero. GLAD generalizes to Defects4J v2.0 with consistent performance (20 correct), unlike prior tools that degrade significantly on the harder v2.0 dataset.", 406 "red_flags": [ 407 { 408 "flag": "No statistical significance testing", 409 "detail": "All comparative claims are based on raw counts of fixed bugs with no significance tests; given small sample sizes (e.g., 8 uniquely fixed bugs), chance cannot be ruled out." 410 }, 411 { 412 "flag": "Non-comparable Defects4J v2.0 baselines", 413 "detail": "Table 4 explicitly notes baseline numbers (TBar, SimFix, Recoder) from Zhu et al. use GZoltar-based FL while GLAD uses perfect FL, making direct comparisons misleading." 414 }, 415 { 416 "flag": "Manual correctness judgment without inter-rater reliability", 417 "detail": "Patches are deemed correct by manual inspection for semantic equivalence to the developer patch, but no inter-rater agreement or second reviewer is mentioned." 418 }, 419 { 420 "flag": "Anonymized code link", 421 "detail": "Code is released via an anonymous review link (anonymous.4open.science) which may not persist long-term, threatening reproducibility." 422 }, 423 { 424 "flag": "No environment specification", 425 "detail": "No requirements.txt, Dockerfile, or equivalent is provided; replication requires guessing dependency versions for javalang, JDB integration, and the GRU training setup." 426 } 427 ], 428 "cited_papers": [ 429 { 430 "title": "TBar: Revisiting Template-Based Automated Program Repair", 431 "relevance": "Primary template-based baseline; GLAD is evaluated against TBar to show orthogonality and ability to fix bugs TBar cannot." 432 }, 433 { 434 "title": "CURE: Code-Aware Neural Machine Translation for Automatic Program Repair", 435 "relevance": "State-of-the-art NMT-based APR baseline used for runtime comparison and as the performance ceiling for learning-based approaches." 436 }, 437 { 438 "title": "Defects4J: A Database of Existing Faults to Enable Controlled Testing Studies for Java Programs", 439 "relevance": "Core evaluation benchmark; the paper's entire empirical analysis is grounded in Defects4J v1.2 and v2.0." 440 }, 441 { 442 "title": "A Syntax-Guided Edit Decoder for Neural Program Repair (Recoder)", 443 "relevance": "Most recent NMT-based APR baseline; compared in Defects4J v2.0 generalization experiment." 444 }, 445 { 446 "title": "Restore: Retrospective Fault Localization Enhancing Automated Program Repair", 447 "relevance": "State-of-the-art condition synthesis baseline used for direct comparison under method-given FL setting." 448 }, 449 { 450 "title": "Precise Condition Synthesis for Program Repair (ACS)", 451 "relevance": "Prior condition synthesis technique directly comparable to GLAD's target; discussed to show GLAD can generate predicates ACS cannot (novel tokens absent from corpus)." 452 }, 453 { 454 "title": "On the Efficiency of Test Suite Based Program Repair (Liu et al. 2020)", 455 "relevance": "Provides the community-level APR results used to establish the 138-fault baseline and motivate the omission fault analysis." 456 }, 457 { 458 "title": "SequenceR: Sequence-to-Sequence Learning for End-to-End Program Repair", 459 "relevance": "Representative NMT-based APR technique establishing the translation paradigm that GLAD moves away from." 460 } 461 ], 462 "engagement_factors": { 463 "practical_relevance": { 464 "score": 2, 465 "justification": "GLAD is a released tool targeting a specific class of bugs (if-statement omissions) with reasonable runtimes, making it deployable as part of an APR ensemble." 466 }, 467 "surprise_contrarian": { 468 "score": 2, 469 "justification": "The insight that omission faults—not deletion/mutation faults—constitute the 'next level' of APR difficulty, backed by systematic analysis of 40 tools, challenges the field's implicit assumptions about what has been 'solved.'" 470 }, 471 "fear_safety": { 472 "score": 0, 473 "justification": "The paper has no AI risk or safety implications; it is an applied software engineering tool." 474 }, 475 "drama_conflict": { 476 "score": 1, 477 "justification": "The paper positions GLAD as fixing bugs that 40 prior tools collectively failed to fix, which creates mild competitive narrative within the APR community." 478 }, 479 "demo_ability": { 480 "score": 2, 481 "justification": "The tool is publicly released and could in principle be applied to any Java project with a test suite; Defects4J is a standard public benchmark for replication." 482 }, 483 "brand_recognition": { 484 "score": 1, 485 "justification": "KAIST is a respected Korean research university with some visibility in SE research, but not a major industry lab driving broad recognition." 486 } 487 }, 488 "hn_data": { 489 "threads": [ 490 { 491 "hn_id": "35623976", 492 "title": "Segment Everything Everywhere All at Once", 493 "points": 3, 494 "comments": 0, 495 "url": "https://news.ycombinator.com/item?id=35623976" 496 }, 497 { 498 "hn_id": "35672099", 499 "title": "Hundred Prisoners and a Lightbulb – Looking Back", 500 "points": 1, 501 "comments": 0, 502 "url": "https://news.ycombinator.com/item?id=35672099" 503 }, 504 { 505 "hn_id": "31948353", 506 "title": "Human-Following and -guiding in Crowded Environments by Mobile Service Robots", 507 "points": 1, 508 "comments": 0, 509 "url": "https://news.ycombinator.com/item?id=31948353" 510 }, 511 { 512 "hn_id": "31773987", 513 "title": "Discrete Contrastive Diffusion for Cross-Modal and Conditional Generation", 514 "points": 1, 515 "comments": 0, 516 "url": "https://news.ycombinator.com/item?id=31773987" 517 } 518 ], 519 "top_points": 3, 520 "total_points": 6, 521 "total_comments": 0 522 } 523 }