ai-research-survey

scan.json (31155B)

---

 {
   "paper": {
     "title": "Revisiting Evolutionary Program Repair via Code Language Model",
     "authors": ["Yunan Wang", "Tingyu Guo", "Zilong Huang", "Yuan Yuan"],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2408.10486",
     "doi": "10.48550/arXiv.2408.10486"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ARJA-CLM combines Code Language Models (CodeLlama, InCoder) with multiobjective evolutionary algorithms for multi-point automated program repair in Java, correctly fixing 56 of 224 Defects4J bugs — 64% more than ARJA-e and 43% more than DEAR. Context-aware prompts that include callable fields and methods improve repair by 21% (56 vs 46 correct patches). On the APR-2024 competition benchmark, ARJA-CLM produced 19 plausible patches (7 correct), outperforming all participating tools. Combining search spaces from different sources before searching does not improve results — searching separately and merging patches is better.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. No promise of future release is made either."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Defects4J v1.0.1, a widely-used public benchmark, and the APR-2024 competition dataset, both publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 mentions hardware (DELL C4140, Intel Xeon Gold 6148, NVIDIA Tesla V100 32GB), OS (Ubuntu 22.04.3 LTS), and Java version (1.7.0_80), but no Python version, ML framework versions, requirements.txt, or dependency specifications are provided. Not enough to recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The approach is described algorithmically but not in a form that enables direct replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 56 correct patches, 94 plausible patches). No confidence intervals or error bars appear in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'ARJA-CLM surpasses all participating repair tools' and '64% more defects' are made by comparing raw counts without any statistical significance test (no p-values, t-tests, or non-parametric tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baseline context are consistently reported: 64% more than ARJA-e (56 vs 34), 43% more than DEAR (56 vs 39), 21% from context-aware prompts (56 vs 46), 280% more plausible patches on APR-2024 (19 vs 5). Raw counts and percentages are both provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 224 bugs from Defects4J and 100 from APR-2024 without justifying why these sizes are adequate. The choice of 224 is inherited from ARJA-e ('following ARJA-e') without independent justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The evolutionary algorithm is stochastic, yet the paper states 'all experiments only perform one round of multiobjective evolution' (Section 4.2). No variance, standard deviation, or spread measures are reported across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 compares ARJA-CLM against 8 baselines: DEAR, HERCULES, Recoder, DLFix, AlphaRepair, GAMMA, TBar, and SimFix."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent tools: DEAR (2022), GAMMA (2023), AlphaRepair (2022), Recoder (2021). The mix of traditional and learning-based tools covers the current landscape well."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 ablates model type and size (CodeLlama-7B vs InCoder-6.7B vs InCoder-1.3B). RQ2 ablates search space sources (CodeLlama vs ARJA-e vs combined). RQ3 ablates context-aware prompts (with vs without extra field/method information)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports plausible patches, correct patches, plausible percentage, correct percentage, and evaluation cost (number of fitness evaluations to find first plausible patch, Table 6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3: 'we manually verify the correctness of the plausible patches by our repair approach.' They inspect the 10 smallest patches for each bug to determine correctness."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Defects4J provides standard test suites not used for tuning. APR-2024 has separate public test suites (visible to repair tools) and private test suites (for verification only), as stated in Section 4.3."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "All result tables (Tables 3-11) break down results by project (Chart, Lang, Math, Time). Table 4 lists individual bug IDs fixed."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows bugs ARJA-e fixes but ARJA-CLM cannot. Section 5.6 discusses limitations with algorithmic bugs: 'the direct context of the single buggy statement is more likely to contain errors besides the erroneous statement itself, misleading the CLM to generate incorrect candidates.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 shows combining search spaces before searching does not improve results (56 correct for CodeLlama alone vs 45 for CodeLlama+ARJA-e). Section 5.3 explains why: 'combining space also exponentially increases the difficulty for the evolutionary algorithm to find the correct patch.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims '64% more defects compared to ARJA-e' matches 56 vs 34 (Table 5). '43% more than DEAR' matches 56 vs 39 (Table 10). 'Context-aware prompts improved repair effectiveness by 21%' matches 56 vs 46 (Table 7). '280% more plausible patches' on APR-2024 matches 19 vs 5 (Table 11). All verified."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The claim that context-aware prompts 'improve' repair is supported by a controlled ablation (RQ3) comparing prompts with and without extra info under identical conditions. The claim that model size improves performance is supported by comparing InCoder-1.3B vs InCoder-6.7B under identical conditions. These are adequate single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly bounds scope: 'our repair approach is aimed at passing the JUnit test suite in Java programs, for which it cannot generalize well to other programming languages or the Java program without test suites.' They also note Defects4J 'cannot fully cover all the defects in the real world.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses data leakage as an alternative explanation for performance, notes that different experimental settings may affect comparisons, discusses prompt trimming as a confound, and addresses GPU variation. They argue context-aware prompts (different from training data) mitigating the leakage explanation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes between plausible patches (pass test suite) and correct patches (manually verified), acknowledging that plausible patches may be overfitting. Section 4.3 states they 'skip the overfitting patches (e.g., those deleting necessary statements).'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific models are named: CodeLlama-7B, InCoder-1.3B, InCoder-6.7B. These are specific enough identifiers with known architectures and parameter counts. Half-precision inference is also stated."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides a concrete example of the full context-aware replacement prompt including the comment structure, field/method information format, FILL_ME marker, and actual code context. Figure 6 shows additional prompt examples. The prompt structure is fully described in Section 3.1."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports sampling parameters (top_p=0.9, top_k=50, temperature=1.0, num_return_sequences=10, max_new_tokens=100). Table 2 reports evolutionary algorithm parameters (N=40, G=50, γ_min=0.1, n_max=60, w=0.5, μ=0.06)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. ARJA-CLM is a traditional pipeline: fault localization → prompt construction → CLM inference → candidate extraction → evolutionary search. No agent loops, tool use, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes prompt construction (context extraction, field/method extraction, trimming to max tokens). Section 3.3 describes code sequence filtering (invalid outputs removed, multi-statement sequences handled, duplicates removed). Section 4.3 describes bug selection (Mockito and Closure excluded with reasons)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to validity' provides substantive discussion divided into 'Internal' and 'External' subsections covering multiple specific concerns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: manual patch verification bias, CodeLlama/InCoder data leakage from GitHub, different experimental settings across compared tools ('this may affect the fairness'), prompt trimming losing information due to memory limits, and GPU-dependent output variation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: 'our repair approach is aimed at passing the JUnit test suite in Java programs, for which it cannot generalize well to other programming languages or the Java program without test suites' and 'the 224 bugs... cannot fully cover all the defects in the real world.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (generated patches, candidate statements, intermediate results) is made available for independent verification. Only aggregate results are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes data sources: Defects4J v1.0.1 with 224 real-world bugs in four projects (Chart, Lang, Math, Time), with reasons for excluding Mockito and Closure. APR-2024 is described as 100 bugs generated by GPT-3.5 and GPT-4 for Leetcode problems."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data sources are standard public benchmarks (Defects4J, APR-2024 competition)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full pipeline: fault localization with Ochiai → LBS extraction → prompt construction with context and field/method info → CLM inference → code sequence filtering and transformation (Figure 3) → candidate set construction → evolutionary search. Figure 4 reports valid statement counts per project."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper. No acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are affiliated with the School of Computer Science and Engineering, Beihang University, Beijing, China. They evaluate third-party open-source models (CodeLlama, InCoder), so no product-affiliation conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funders cannot be assessed. The absence of any funding disclosure means this criterion is not satisfiable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for CodeLlama and InCoder are not stated in the paper, despite both being trained on GitHub data that may overlap with Defects4J."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6 and Section 7 explicitly discuss this: 'We notice the possibility of source code from Defect4J leaking into CodeLlama which we uses.' They argue context-aware prompts make prompts 'different from training data' and test on APR-2024 to mitigate."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "They address contamination by conducting additional experiments on the newly released APR-2024 competition dataset (Section 5.6) and arguing that 'the buggy programs in the newly released APR-2024 AI Track benchmark are written by GPT, with less data leakage issues.' Results on APR-2024 confirm performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of an automated program repair system."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of an automated program repair system."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of an automated program repair system."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of an automated program repair system."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of an automated program repair system."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of an automated program repair system."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of an automated program repair system."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 states 'the time of one round of search is within 1 hour' but this is a vague upper bound. No per-bug inference time, token counts, or monetary costs are reported. Table 6 reports evaluation cost as number of fitness evaluations, not wall-clock time or compute cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (V100 32GB, Intel Xeon Gold 6148) but total GPU hours, total experiment time, or overall computational budget are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The evolutionary algorithm is inherently stochastic, yet only 'one round of multiobjective evolution' is performed per experiment (Section 4.2). No analysis of sensitivity to random seeds or initial populations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 explicitly states: 'All experiments only perform one round of multiobjective evolution on the candidate statements.' The number of runs (1) is clearly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Evolutionary algorithm parameters are inherited from ARJA-e (Table 2) and sampling parameters are set without justification (Table 1). No hyperparameter search or budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper transparently compares all three models (RQ1) and all search space configurations (RQ2), showing all results in Tables 3 and 5. CodeLlama-7B is selected as best based on comprehensive comparison, not cherry-picking."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 acknowledges: 'our experimental results for comparison come directly from the papers whose experimental settings are different, this may affect the fairness of the repair effect comparison.' They also note that AlphaRepair and GAMMA use perfect fault localization while ARJA-CLM does not, disadvantaging their own system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute budget. The search cost (Table 6) reports fitness evaluations but not compute time, and there is no comparison of compute requirements across methods."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J is used as a standard benchmark without any discussion of whether its 224 bugs represent real-world defect distributions or whether passing test suites constitutes genuine correctness."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The pipeline (fault localization + CLM + evolutionary search) is the system under test, not a scaffold around a model."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses that Defects4J source code may have leaked into CodeLlama's training data (Section 7) and addresses this by testing on the newly released APR-2024 dataset with AI-generated bugs less likely to be in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (e.g., fault localization information, test suite feedback during search) provides information beyond what would be available in real deployment scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Defects4J bugs from the same project (e.g., multiple Math bugs) are independent, or whether project-level correlations could inflate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap) is applied. The mitigation strategy of using APR-2024 is a workaround, not a detection method."
       }
     }
   },
   "claims": [
     {
       "claim": "ARJA-CLM correctly fixes 56 of 224 Defects4J bugs, 64% more than ARJA-e (34) and 43% more than DEAR (39).",
       "evidence": "Tables 4, 5, and 10 report correct patch counts. Table 10 provides per-project breakdown showing ARJA-CLM surpasses all compared tools.",
       "supported": "strong"
     },
     {
       "claim": "Context-aware prompts that include callable fields and methods improve repair effectiveness by 21%.",
       "evidence": "Table 7 shows 56 correct patches with extra info vs 46 without. Table 8 identifies 12 correct patches using extra info, 7 of which are unfixable without it. Figure 6 shows concrete examples.",
       "supported": "moderate"
     },
     {
       "claim": "ARJA-CLM generates 280% more plausible patches than ARJA-e on the APR-2024 benchmark (19 vs 5), surpassing all participating tools.",
       "evidence": "Table 11 compares ARJA-CLM against 7 other tools on 100 APR-2024 bugs. ARJA-CLM produces 19 plausible and 7 correct patches.",
       "supported": "moderate"
     },
     {
       "claim": "Better overall model performance (HumanEval scores) correlates with better candidate statement generation for APR.",
       "evidence": "Section 5.2 compares CodeLlama-7B (HumanEval 33.5%) vs InCoder-6.7B (15.2%). Figure 4 shows CodeLlama generates more valid statements across all projects. Table 3 shows CodeLlama produces more plausible patches (94 vs 71).",
       "supported": "moderate"
     },
     {
       "claim": "Combining search spaces before evolutionary search does not improve repair effectiveness compared to using CodeLlama alone.",
       "evidence": "Table 5 shows CodeLlama alone yields 56 correct patches vs 45 for CodeLlama+ARJA-e combined. Section 5.3 explains that combining exponentially increases search difficulty.",
       "supported": "strong"
     },
     {
       "claim": "ARJA-CLM fixes 22 unique bugs that no other compared tool can fix.",
       "evidence": "Figure 7 Venn diagram shows 22 bugs unique to ARJA-CLM when compared against DEAR, GAMMA, Recoder, and HERCULES.",
       "supported": "strong"
     },
     {
       "claim": "ARJA-CLM repairs more multilocation bugs than ARJA-e (17 correct vs 10 correct).",
       "evidence": "Table 9 breaks down multilocation bug fixes by project, showing ARJA-CLM generates plausible patches for 38 and correct patches for 17 multilocation bugs vs 32/10 for ARJA-e.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Single stochastic run without variance",
       "detail": "The evolutionary algorithm is inherently stochastic, yet all experiments use only one round of evolution (Section 4.2). Without multiple runs, it is impossible to know whether the reported counts are typical or lucky outcomes. This is especially concerning for the APR-2024 results where the margins are small (7 vs 5 correct patches)."
     },
     {
       "flag": "Unfair baseline comparison conditions",
       "detail": "Section 7 acknowledges that baseline results come from other papers with different experimental settings. AlphaRepair and GAMMA use perfect fault localization while ARJA-CLM uses imperfect (Ochiai), making the comparison non-uniform. While this actually disadvantages ARJA-CLM, it prevents apples-to-apples conclusions."
     },
     {
       "flag": "No code released",
       "detail": "Despite proposing a novel tool (ARJA-CLM), no source code is released or linked. This prevents independent verification of the reported results and limits practical impact."
     },
     {
       "flag": "Manual patch verification subjectivity",
       "detail": "Correctness of plausible patches is determined by manual inspection. Section 7 acknowledges: 'Due to the limitations of knowledge and the complexity of defects, we may misjudge the correctness of the patch.' No inter-rater agreement or independent verification is reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair is a key CLM-based APR baseline that uses CodeBERT for zero-shot program repair, directly compared against ARJA-CLM."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "CodeLlama is the primary code language model used in ARJA-CLM for generating candidate repair statements."
     },
     {
       "title": "Incoder: A generative model for code infilling and synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan", "Jessy Lin"],
       "year": 2022,
       "arxiv_id": "2204.05999",
       "relevance": "InCoder is one of the code language models evaluated in ARJA-CLM for code infilling in program repair."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained code model used in related APR work (AlphaRepair), relevant to understanding CLM capabilities for code tasks."
     },
     {
       "title": "GAMMA: Revisiting template-based automated program repair via mask prediction",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Tongke Zhang"],
       "year": 2023,
       "relevance": "State-of-the-art CLM-based APR using mask prediction, compared against ARJA-CLM. Uses perfect fault localization."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark used to compare overall model performance between CodeLlama and InCoder."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "arxiv_id": "2302.05020",
       "relevance": "Directly studies the impact of CLMs on APR effectiveness, closely related to this paper's research questions."
     },
     {
       "title": "Practical program repair in the era of large pre-trained language models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2022,
       "arxiv_id": "2210.14179",
       "relevance": "Studies practical aspects of using pre-trained LLMs for program repair, addressing similar challenges as ARJA-CLM."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Zhiyu Fan", "Xiang Gao", "Martin Mirchev", "Abhik Roychoudhury", "Shin Hwei Tan"],
       "year": 2023,
       "relevance": "Explores automated repair of LLM-generated code, relevant to understanding CLM reliability and repair needs."
     },
     {
       "title": "DeepCode AI Fix: Fixing security vulnerabilities with large language models",
       "authors": ["Berkay Berabi", "Alexey Gronskiy", "Veselin Raychev"],
       "year": 2024,
       "arxiv_id": "2402.13291",
       "relevance": "Uses LLMs for fixing security vulnerabilities, extending APR to the safety/security domain."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq Joty", "Steven C.H. Hoi"],
       "year": 2021,
       "arxiv_id": "2109.00859",
       "relevance": "Pre-trained code model relevant to the broader landscape of code language models used in software engineering tasks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Proposes a concrete tool for multi-point automated program repair in Java, but no code is released limiting immediate practical use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Combining evolutionary algorithms with CLMs is a novel integration but the finding that larger/better models help is expected; the negative result about merging search spaces is mildly surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security implications; this is a software engineering tool for fixing bugs."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward empirical comparison of repair tools."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code repository, demo, or installable tool is provided."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Beihang University; uses open-source models (CodeLlama, InCoder) rather than products from famous AI labs."
     }
   }
 }