ai-research-survey

scan.json (29739B)

---

 {
   "paper": {
     "title": "Neural Program Repair with Program Dependence Analysis and Effective Filter Mechanism",
     "authors": [
       "Yuwei Zhang",
       "Ge Li",
       "Zhi Jin",
       "Ying Xing"
     ],
     "year": 2023,
     "venue": "arXiv.org",
     "arxiv_id": "2305.09315",
     "doi": "10.48550/arXiv.2305.09315"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "REPEATNPR combines program dependence graph slicing with an ensemble filter mechanism for neural program repair of single-line Java bugs. On five benchmarks, it outperforms 10 baselines using exact match: 2540 correct patches on BFP (vs 2191 for CodeT5, a 15.9% gain) and similar improvements on Bugs.jar, Defects4J, Bears, and QuixBugs. Ablation shows PDG-based context extraction contributes more than the filter mechanism, and both components are independently beneficial across all pre-trained models tested.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper. The implementation uses PyTorch and CodeT5-small but no code release is mentioned."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: BFP [11], Bugs.jar [53], Defects4J [54], Bears [55], and QuixBugs [56]. These are standard public benchmarks that were not modified in a way that requires separate release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions PyTorch, CodeT5-small from HuggingFace, PROGEX, and 4 Nvidia GTX 1080Ti GPUs, but provides no requirements.txt, Docker file, or detailed dependency listing with library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology is described in prose but there are no concrete commands or execution steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables II–VI and Figure 4 are reported as point estimates (raw counts or percentages) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims REPEATNPR 'substantially outperforms' and 'significantly outperforms' baselines (Section V-A) based solely on comparing raw numbers. No statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context: 'REPEATNPR produces more correct patches than the best baseline model CodeT5 by 15.9% in the BFP benchmark, 12.0% in the Bugs.jar benchmark, 22.2% in the Defects4J benchmark, 62.5% in the Bears benchmark, 7.1% in the QuixBugs benchmark' (Section V-A1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes or power analysis. The BFP dataset size (141k training) and benchmark sizes are stated but not justified. Some benchmarks are very small (QuixBugs: 32 bugs, Bears: 119 bugs) without discussion of statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. All results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Ten baselines are compared: CODIT, Edits, Tufano, Recoder, CoCoNut, SEQUENCER, RoBERTa, CodeBERT, GraphCodeBERT, and CodeT5 (Section IV-B1, Table II)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines span 2019–2022 (CodeT5 2021, Recoder 2021, GraphCodeBERT 2021, SEQUENCER 2021, CoCoNut 2020), which are contemporary for a 2023 paper in the neural program repair space."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table III presents an ablation study removing program dependence analysis (A) and filter mechanism (F) independently. Table V further investigates model order and quantity in the filter mechanism."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports exact match counts (Table II), Fix@k for k=1,5,10 (Tables IV-V, Figure 4), and per-bug-type breakdown (Table VI with Simple Delete, Simple Insert, Simple Replace, Mixed)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The paper explicitly uses exact match to 'avoid human bias and reduce manual effort' (Section IV-B3), acknowledging in the threats section that they plan future human evaluations."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The BFP dataset is split 8:1:1 into training, validation, and testing sets (Section IV-A). Results are reported on the test set. Model selection uses validation loss. Four additional benchmarks serve as fully external test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table VI provides per-bug-type breakdowns (Simple Delete, Simple Insert, Simple Replace, Mixed) on BFP. Table II provides per-benchmark breakdowns. Figure 7 shows overlapping patching rates."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The unaltered patching phenomenon is identified and analyzed as a systematic failure mode (Section I, Fig. 1). Figure 5 shows a specific example where ablated models fail but REPEATNPR succeeds. Section V-C2 discusses the overlapping phenomenon among model outputs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table V shows that changing model order in the filter mechanism decreases performance (FSRA+CodeT5A underperforms FCodeT5A+SRA). Adding a third model (GCBA) yields diminishing returns. Section V-C2 discusses overlap limitations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that REPEATNPR 'demonstrates effectiveness on five benchmarks when compared with state-of-the-art baselines' are supported by Table II showing improvements across all five benchmarks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions ('component A improves performance') are supported by controlled ablation experiments in Table III, where single components are added/removed while other factors remain constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states scope: 'REPEATNPR framework focuses on single-line bugs written in the Java programming language' (Section III). The abstract specifies 'fixing single-line Java bugs.' Threats section acknowledges Java-only evaluation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section VI discusses threats to validity (hyper-parameter sensitivity, exact match metric, generalizability) but does not substantively discuss alternative explanations for the observed improvements. For example, the possibility that improvements come from additional input tokens rather than PDG structure is not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes its proxy (exact match to human-written patch) from true correctness in Section IV-B3: 'Although such a metric does not represent human judgment, it is a strict and objective metric.' The construct threat in Section VI further acknowledges this gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'CodeT5-small' with the exact HuggingFace checkpoint URL (https://huggingface.co/Salesforce/codet5-small) in Section IV-B2. For baselines, publicly released checkpoints or source codes are used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper fine-tunes CodeT5 rather than using prompting. The input is tokenized code sequences with special tokens, not natural language prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B2 reports: max sequence length 512, batch size 32, beam size 10, candidate number 10, max 20 epochs, early stopping patience 5, 8-headed attention, 6 layers in encoder and decoder. Uses same hyperparameters as Raffel et al. [59]."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. REPEATNPR is a fine-tuned encoder-decoder model with a post-processing filter step, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A and IV-A describe preprocessing: PROGEX parses Java methods into PDGs, instances failing parsing are filtered out, sequences longer than 512 after subword tokenization are truncated, dataset split 8:1:1 ensuring no cross-repository contamination. Table I provides final statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' provides a dedicated section with substantive discussion of external, internal, and construct threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: hyper-parameter sensitivity with acknowledgment of computational resource constraints preventing thorough exploration, exact match not representing human judgment, evaluation limited to Java, and quality of the BFP dataset as the primary training source."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Scope is clearly bounded: single-line Java bugs only, evaluated with exact match metric, fine-tuned on BFP dataset. Section VII explicitly notes plans to 'extend REPEATNPR with small modifications to support new target languages' indicating current Java-only scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available: BFP [11], Bugs.jar [53], Defects4J [54], Bears [55], QuixBugs [56]. However, the authors' specific adapted dataset (after PDG processing and filtering) is not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "BFP dataset origin is described: '787k bug-fixing commits extracted from the GitHub repositories' (Section IV-A). The adaptation process (PDG parsing, filtering, truncation, splitting) is documented with final counts in Table I."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks with documented collection procedures in their original papers."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: BFP 787k commits → PROGEX PDG parsing (filtering failures) → truncation at 512 tokens → 8:1:1 split with repository-level dedup → final counts in Table I (141195 train, 13523 validation, 13635 test)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding, acknowledgments, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Peking University (Key Laboratory of High Confidence Software Technologies) and Beijing University of Posts and Telecommunications."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper evaluates the authors' own framework, not a commercial product."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state CodeT5's pre-training data cutoff date. CodeT5 was pre-trained on a 'colossal clean crawled corpus' but no temporal boundary is given for what code it may have seen."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A explicitly addresses overlap: 'we remove all duplicate instances between the training and testing sets' and 'any two instances belonging to the same GitHub repository cannot be put in two different sets.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether CodeT5's pre-training data contained code from BFP, Bugs.jar, Defects4J, Bears, or QuixBugs benchmarks. These are all publicly available and could be in the pre-training corpus."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Evaluation is entirely automated using exact match on benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example timing is reported. The paper does not mention how long inference takes or how many patches can be generated per unit time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is stated (4 Nvidia GTX 1080Ti GPUs, 12GB each) in Section IV-B2, but total training time, GPU hours, or compute budget is not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from a single run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented as single values without any indication of how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper acknowledges: 'Due to the limitation of computational resources, we cannot thoroughly explore optimal hyper-parameters' (Section VI) and uses settings from Raffel et al. [59], but does not state what, if any, search was performed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Model selection is done on the validation set: 'After each epoch, we compute the loss on the validation set and save the model with the minimum validation loss' (Section IV-B2)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across 10 baselines and 5 benchmarks but uses no statistical tests at all, let alone correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors retrain all baselines 'via the same training strategy and hyper-parameter settings' (Section V-A) which may disadvantage methods designed for different configurations. This potential bias is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "REPEATNPR uses an ensemble of two models (CodeT5 + SEQUENCER) but the additional compute cost compared to single-model baselines is not discussed or quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B3 and Section VI (construct threat) discuss that exact match 'does not represent human judgment' and is 'a strict and objective metric.' The paper acknowledges the limitation and plans future human evaluations."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are fine-tuned encoder-decoder models evaluated directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal aspects of the BFP dataset or whether training data commits predate test data commits. No temporal split is mentioned."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the input context (PDG slices, method signatures) could leak information about the correct fix in ways that would not be available in a real debugging scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A explicitly states: 'any two instances belonging to the same GitHub repository cannot be put in two different sets (e.g., one in training and the other in testing).' This addresses non-independence from shared repository origins."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Repository-level splitting is applied as a concrete prevention method: instances from the same GitHub repository are kept in the same split. Duplicate removal between train and test sets is also performed."
       }
     }
   },
   "claims": [
     {
       "claim": "REPEATNPR outperforms 10 state-of-the-art baselines on all five benchmarks in exact match metric.",
       "evidence": "Table II shows REPEATNPR generates 2540 correct patches on BFP (vs 2191 for CodeT5), 168 on Bugs.jar (vs 150), 44 on Defects4J (vs 36/38), 26 on Bears (vs 16), and 15 on QuixBugs (vs 14/15). Section V-A1.",
       "supported": "moderate"
     },
     {
       "claim": "Program dependence analysis improves bug-fixing performance of all pre-trained models tested.",
       "evidence": "Table IV shows Fix@10 improvements: RoBERTa 11.23%→11.82%, CodeBERT 13.79%→14.24%, GraphCodeBERT 13.38%→13.92%, CodeT5 17.92%→19.89%. Section V-B2.",
       "supported": "moderate"
     },
     {
       "claim": "The filter mechanism increases correct patches by eliminating unaltered patches and passing them to a second model.",
       "evidence": "Table III shows CodeT5A generates 2431 patches on BFP while REPEATNPR (CodeT5A + filter) generates 2540, a 4.5% improvement. Gains are consistent across all five benchmarks. Section V-B1.",
       "supported": "moderate"
     },
     {
       "claim": "Pre-trained models are more promising than models trained from scratch for neural program repair.",
       "evidence": "Table II shows CodeT5 (pre-trained) generates 2191 patches vs SEQUENCER (trained from scratch) at 1917 on BFP. Section V-A1 notes 'CodeT5 improves SEQUENCER by 14.3% in the BFP benchmark and 51.5% in the Bugs.jar benchmark.'",
       "supported": "moderate"
     },
     {
       "claim": "The order of models in the filter mechanism affects performance.",
       "evidence": "Table V shows FCodeT5A+SRA achieves Fix@10 of 20.78% while FSRA+CodeT5A achieves only 17.23% on BFP. Section V-B3.",
       "supported": "moderate"
     },
     {
       "claim": "REPEATNPR generates 390 unique correct patches that no other model can produce.",
       "evidence": "Figure 7 diagonal shows REPEATNPR has 390 unique patches on BFP, second only to Recoder. Section V-C2.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "All experimental results are reported as single-point estimates. No standard deviation, confidence intervals, or multiple-run results are provided, making it impossible to assess result stability. This is particularly concerning for DL models known to be sensitive to initialization."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of 'significantly outperforms' and 'substantially outperforms' are made based solely on comparing raw numbers. With some benchmarks having very small sizes (QuixBugs: 32, Bears: 119), differences may not be statistically meaningful."
     },
     {
       "flag": "Baselines retrained with unified settings",
       "detail": "All baselines are retrained 'via the same training strategy and hyper-parameter settings' (Section V-A). Methods like CoCoNut and Recoder were designed with specific architectures and training procedures; forcing them into uniform settings may systematically disadvantage them."
     },
     {
       "flag": "No code released",
       "detail": "Despite proposing a novel framework with multiple components (PDG analysis, filter mechanism), no source code is provided, preventing independent verification or replication."
     },
     {
       "flag": "Small benchmark sizes for some evaluation sets",
       "detail": "QuixBugs has only 32 bugs and Bears 119 bugs. Differences of 1-2 patches on QuixBugs (e.g., 14 vs 15) are presented as improvements but are not statistically meaningful at this scale."
     }
   ],
   "cited_papers": [
     {
       "title": "An empirical study on learning bug-fixing patches in the wild via neural machine translation",
       "authors": ["M. Tufano", "C. Watson", "G. Bavota", "M. D. Penta", "M. White", "D. Poshyvanyk"],
       "year": 2019,
       "relevance": "Foundational neural program repair dataset (BFP) and NMT-based approach used as the primary training data source and baseline."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Y. Wang", "W. Wang", "S. R. Joty", "S. C. H. Hoi"],
       "year": 2021,
       "relevance": "Pre-trained code model used as the backbone architecture for REPEATNPR; demonstrates transfer learning effectiveness for code tasks."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang", "N. Duan", "X. Feng", "M. Gong"],
       "year": 2020,
       "relevance": "Bimodal pre-trained code model used as a baseline; early example of pre-training for code understanding."
     },
     {
       "title": "GraphCodeBERT: Pre-training code representations with data flow",
       "authors": ["D. Guo", "S. Ren", "S. Lu", "Z. Feng", "D. Tang", "S. Liu"],
       "year": 2021,
       "relevance": "Graph-based pre-trained code model incorporating data flow; used as baseline and shows relevance of structural code features."
     },
     {
       "title": "CoCoNut: Combining context-aware neural translation models using ensemble for program repair",
       "authors": ["T. Lutellier", "H. V. Pham", "L. Pang", "Y. Li", "M. Wei", "L. Tan"],
       "year": 2020,
       "relevance": "Ensemble-based neural program repair approach; directly relevant as it combines multiple models for bug fixing, similar to REPEATNPR's filter mechanism."
     },
     {
       "title": "CURE: code-aware neural machine translation for automatic program repair",
       "authors": ["N. Jiang", "T. Lutellier", "L. Tan"],
       "year": 2021,
       "relevance": "Code-aware NMT for program repair using pre-trained GPT module for contextual embeddings."
     },
     {
       "title": "Recoder: A syntax-guided edit decoder for neural program repair",
       "authors": ["Q. Zhu", "Z. Sun", "Y. Xiao", "W. Zhang", "K. Yuan", "Y. Xiong", "L. Zhang"],
       "year": 2021,
       "relevance": "Syntax-guided program repair using AST-based edit operations; contrasting approach to NMT-based methods."
     },
     {
       "title": "Sequencer: Sequence-to-sequence learning for end-to-end program repair",
       "authors": ["Z. Chen", "S. Kommrusch", "M. Tufano", "L. Pouchet", "D. Poshyvanyk", "M. Monperrus"],
       "year": 2021,
       "relevance": "End-to-end Seq2Seq program repair with copy mechanism; used as the secondary model in REPEATNPR's filter mechanism."
     },
     {
       "title": "Less training, more repairing please: Revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "Explores zero-shot LLM-based program repair, contrasting with the fine-tuning approach taken by REPEATNPR."
     },
     {
       "title": "Standup4NPR: Standardizing setup for empirically comparing neural program repair systems",
       "authors": ["W. Zhong", "H. Ge", "H. Ai", "C. Li", "K. Liu", "J. Ge", "B. Luo"],
       "year": 2022,
       "relevance": "Standardized evaluation framework for comparing NPR systems; addresses the reproducibility crisis in NPR research."
     },
     {
       "title": "Patching as translation: the data and the metaphor",
       "authors": ["Y. Ding", "B. Ray", "P. T. Devanbu", "V. J. Hellendoorn"],
       "year": 2020,
       "relevance": "Edit-based neural program repair model performing token-level operations; alternative to sequence generation approaches."
     },
     {
       "title": "A survey of learning-based automated program repair",
       "authors": ["Q. Zhang", "C. Fang", "Y. Ma", "W. Sun", "Z. Chen"],
       "year": 2023,
       "arxiv_id": "2301.03270",
       "relevance": "Contemporary survey of learning-based program repair covering the landscape that REPEATNPR contributes to."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Automated bug fixing is practically useful but no code is released, and the approach requires fine-tuning infrastructure with GPUs."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms expected trends: pre-training helps, more context helps, ensembles help. No surprising or contrarian findings."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; focuses on improving automated program repair."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Standard incremental improvement paper with no controversial claims or conflict."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released. Cannot be tried by practitioners."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Peking University, a respected institution but not a high-profile AI lab brand for general tech audiences."
     }
   }
 }