ai-research-survey

scan.json (30204B)

---

 {
   "paper": {
     "title": "Exploring and Lifting the Robustness of LLM-powered Automated Program Repair with Metamorphic Testing",
     "authors": [
       "Pengyu Xue",
       "Linhao Wu",
       "Zhen Yang",
       "Zhongxing Yu",
       "Zhi Jin",
       "Ge Li",
       "Yan Xiao",
       "Shuo Liu",
       "Xinyi Li",
       "Hongyi Lin",
       "Jingwen Wu"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2410.07516",
     "doi": "10.48550/arXiv.2410.07516"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "MT-LAPR, a metamorphic testing framework with nine code perturbation rules across token, statement, and block levels, reveals that 34.4%–48.5% of test cases expose instability in four LLMs' automated program repair capabilities. A negative correlation between perturbation distance and repair performance suggests that code readability positively affects LAPR robustness. A CodeT5-based readability improvement model, embedded as a preprocessing step, enhances LAPR robustness by up to 49.32% without modifying the LLMs' parameters.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to the MT-LAPR implementation is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The base datasets (Defects4J and QuixBugs) are standard public benchmarks. The perturbations are deterministic AST transformations fully described in the paper, making reconstruction feasible from public data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or library versions are provided. The paper mentions JavaParser and Python difflib but gives no version or setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The perturbation algorithms are described at a conceptual level but full implementation details are absent."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (R-scores in Tables II–VI) are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Spearman correlation with p-value for the edit-distance analysis (Section V-D) and kappa coefficients for inter-rater agreement, but the main comparative claims about model performance differences (e.g., Mistral Large vs LLaMA3-70B) lack significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Improvements are reported with baseline context: e.g., LLaMA3-8B R-score from 0.440 to 0.657 (49.32% improvement), and percentage declines per model are quantified in Tables II and V."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 60 base samples per dataset (15 per LLM) with no justification for why 15 is sufficient, no power analysis, and no discussion of whether this sample size supports generalizable conclusions."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Temperature=0 makes outputs deterministic, but this eliminates only inference randomness — variance across different base sample selections is not assessed."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baseline robustness testing framework or alternative perturbation approach is compared against MT-LAPR. The paper evaluates MT-LAPR in isolation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included, so contemporaneity cannot be assessed."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Table III) evaluates each of the 9 MRs individually, and RQ3 (Figure 2) studies the effect of combining different numbers of MRs (perturbation distance), effectively ablating components of the framework."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses R-score as the primary metric, supplemented by edit distance analysis (Table III), readability scores (Figure 2), and kappa coefficients for agreement assessment."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Developer surveys are included: RQ1 (Section V-A) has 10 developers rate perturbation frequency on a Likert scale, and RQ3 includes readability assessments by developer pairs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For the CodeT5 fine-tuning (Section VI-A), the paper states: 'we still test on the dataset used in previous experiments (RQ2~5), while preparing the training dataset with the rest of the samples,' establishing a proper train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by perturbation rule (Table III), repair pattern (Table IV), perturbation distance (Figure 2), and per-model (Table II)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ5 (Table IV) identifies repair patterns where LLMs are most vulnerable (Missing Null-Check at 0.298 R-score), and Section VI-C discusses approaches that failed (LLM refactoring causing 20.5% R-score reduction)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-C 'Trial and error' reports that direct LLM code refactoring led to a 20.5% reduction in R-score, and that manual reverse rule design has scalability limitations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '34.4%~48.5% instability' match Table II averages (1-0.656=34.4%, 1-0.515=48.5%). The '49.32% improvement' claim matches Table V (LLaMA3-8B with CodeT5-large)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The perturbation study uses controlled single-variable manipulation (same code, only perturbation differs, temperature=0 eliminates inference randomness), which is adequate for the causal claim that perturbations cause repair failures. The readability improvement experiment similarly isolates the preprocessing step."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims broadly about 'LLM-powered Automated Program Repair' but experiments are restricted to Java only, 4 specific LLMs, and 2 datasets. The threats section acknowledges the Java limitation but the title and abstract do not bound the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses data leakage as an alternative explanation and runs a dedicated experiment (Table VI). The paper also considers whether edit distance rather than perturbation content drives results (Section V-D, finding no significant correlation)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "R-score directly measures what is claimed: the proportion of perturbed test cases on which the LLM still produces a correct repair. The proxy aligns with the measured outcome (robustness to perturbation)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified as 'Mistral Large', 'LLaMA3-70B', 'LLaMA3-8B', and 'CodeGemma-7B' without snapshot dates, API versions, or exact model identifiers. 'Mistral Large' is especially vague as Mistral has released multiple versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the parameter settings and prompt templates for all LLMs reviewed are fixed the same' (Section IV-A) but does not provide the actual prompt text used for repair."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature=0 is stated for all LLMs (Section IV-A). For CodeT5 fine-tuning: 3 epochs, learning rate 5×10⁻⁵, batch size 1, weight decay 0.01 (Section VI-A)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. LLMs are called directly for code repair without agent loops, tools, or iterative workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B documents the base sample construction: filter for successfully repaired samples, follow Sobreira et al.'s taxonomy to sample across repair patterns, obtain 60 samples per dataset. Section IV-D details test case construction with combinatorial sampling."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threats to Validity' provides substantive discussion of internal, external, and construct validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section identifies specific issues: the nine MRs may not cover all coding discrepancies, Java-only limits generality, Defects4J data leakage risk (with dedicated experiment), and test suite evaluation may miss literal inconsistencies."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VII explicitly states that focusing on Java limits generality, the MRs may not encapsulate all real-world coding styles, and the selected LLMs may affect conclusions. Future work to extend to other PLs is noted."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (perturbed test cases, LLM outputs, repair results) is released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The pilot study describes collecting 500 Codeforces samples (10 problems, 50 Java codes each, average difficulty, highest passing rate). The base sample selection from Defects4J and QuixBugs is documented in Section IV-B."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '10 full-time Java developers (at least 3-5 years of coding experience) from the industry' but does not describe how they were recruited, from which companies, or whether recruitment could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: original datasets → filter successfully repaired samples → select across repair patterns (60 base samples) → apply perturbation rules → construct single/multi-distance test cases with combinatorial sampling → evaluate via test suites with reverse processing for variable/method names."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Shandong University, Peking University, Nanyang Technological University, Sun Yat-sen University, and City University of Hong Kong."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, making independence impossible to assess."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest declarations appear in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four LLMs (Mistral Large, LLaMA3-70B/8B, CodeGemma-7B)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII explicitly discusses data leakage: 'since the datasets we used have been widely studied, data leakage may pose an internal threat.' A dedicated experiment (Table VI) uses perturbed samples as leakage-free test sets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII constructs Defects4J_free-leakage and QuixBugs_free-leakage datasets from perturbed samples that LLMs are unlikely to have seen during pre-training, and re-runs experiments to validate conclusions hold (Table VI)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the developer surveys."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the developer surveys."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Survey respondents are characterized as 'ten full-time Java developers (at least 3-5 years of coding experience for each) from the industry' (Section IV-D1)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Inclusion criterion is stated: 'at least 3-5 years of practical Java development experience' and 'full-time Java developers from the industry' (Sections III-A, IV-D1)."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human component is a cross-sectional survey/rating task, not an experimental study with conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "The human component is a Likert-scale rating task on perturbation frequency and readability, not an experimental comparison where blinding would apply."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information about whether all 10 developers completed all survey/rating tasks or whether any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, or latency is reported despite running 4 LLMs across thousands of test cases."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, total API spend, or training time is reported for either the LLM evaluations or the CodeT5 fine-tuning."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are not reported across multiple random seeds. Temperature=0 is used to make outputs deterministic, but the paper does not report sensitivity to other sources of variance (e.g., base sample selection)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Temperature=0 implies single-run deterministic outputs, but this is not stated as '1 run per test case.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported for the CodeT5 fine-tuning (learning rate, batch size, epochs appear fixed without search). LLM temperature is fixed at 0 without justification."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Temperature=0 is stated without justification for why this is the best setting. CodeT5 hyperparameters are reported but their selection is not justified."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across 4 models, 9 perturbation rules, 9 perturbation distances, and 9 repair patterns, but no correction for multiple comparisons is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own MT-LAPR framework without acknowledging the bias of self-evaluation or using independent evaluators for the framework's effectiveness."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis is provided. Model sizes range from 7B to unknown (Mistral Large) but cost/compute differences are not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes that QuixBugs contains simpler lab bugs vs Defects4J's real-world bugs, but does not substantively discuss whether these benchmarks adequately measure LAPR robustness or their construct validity limitations."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; LLMs are called directly for repair."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses that Defects4J and QuixBugs have been widely studied and may appear in training data. A dedicated experiment (Table VI) creates leakage-free datasets from perturbed samples to assess temporal leakage impact."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the repair prompt format or surrounding context could leak information about the expected fix."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples share structural similarities (e.g., same projects in Defects4J appearing across samples)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper constructs 'free-leakage' datasets (Defects4J_free-leakage, QuixBugs_free-leakage) using perturbed samples that LLMs are unlikely to have encountered during pre-training, then re-evaluates performance (Table VI)."
       }
     }
   },
   "claims": [
     {
       "claim": "34.4%–48.5% of MT-LAPR test cases expose the instability of LAPR techniques on average across four LLMs.",
       "evidence": "Table II shows average R-scores of 0.515 (Defects4Jtest) and 0.656 (QuixBugstest), meaning 48.5% and 34.4% of test cases caused repair failure after perturbation. Per-model R-scores range from 0.248 (CodeGemma on QuixBugs) to 0.951 (Mistral Large on QuixBugs).",
       "supported": "strong"
     },
     {
       "claim": "There is a significant negative correlation between perturbation distance and LLMs' program repair performance.",
       "evidence": "Figure 2 shows R-score declining as perturbation distance increases from 1 to 9 on both datasets. The trend is consistent across all models.",
       "supported": "moderate"
     },
     {
       "claim": "Code readability positively correlates with LAPR robustness.",
       "evidence": "Figure 2 shows readability scores (from developer surveys) decreasing in parallel with R-score as perturbation distance increases. Cohen's kappa for inter-rater agreement is 0.65 and 0.67 (Section V-C).",
       "supported": "moderate"
     },
     {
       "claim": "CodeT5-based readability improvement enhances LAPR robustness by up to 49.32%.",
       "evidence": "Table V: LLaMA3-8B R-score improved from 0.440 to 0.657 with CodeT5-large (49.32% improvement). LLaMA3-70B improved from 0.536 to 0.617 (15.11%). Both CodeT5 models reduced edit distances of ~75% of perturbed codes.",
       "supported": "strong"
     },
     {
       "claim": "All nine proposed MRs are prevalent and practically significant among developers.",
       "evidence": "Figure 1 shows average frequency scores >3 (Generally Frequent) for all nine perturbations from 10 industry developers. Randolph's Kappa = 0.76 indicating 'almost perfect agreement' (Section V-A).",
       "supported": "weak"
     },
     {
       "claim": "Perturbation content is more significant than the number of perturbed tokens in affecting LLM repair performance.",
       "evidence": "Section V-D: Spearman correlation between edit distances and R-scores is 0.042 with p=0.914, showing no significant relationship between the amount of perturbation and LLM performance.",
       "supported": "moderate"
     },
     {
       "claim": "Larger LLMs exhibit better perturbation resistance in APR (scaling effect).",
       "evidence": "Table II: LLaMA3-70B (R-score 0.536/0.757) consistently outperforms LLaMA3-8B (0.440/0.668) on both datasets. Mistral Large performs best overall.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small base sample sizes",
       "detail": "Only 15 base samples per LLM (60 total per dataset) are used, selected from samples each LLM can already repair. This is a very small foundation for claims about LAPR robustness in general. The combinatorial expansion to construct test cases does not compensate for the narrow base."
     },
     {
       "flag": "No code or framework released",
       "detail": "Neither the MT-LAPR implementation, the generated test cases, the CodeT5 models, nor the experimental outputs are released, preventing independent verification or replication."
     },
     {
       "flag": "Survey with only 10 developers",
       "detail": "Claims about perturbation prevalence and code readability rely on surveys with only 10 industry developers. This is too small to generalize about developer practices broadly, and recruitment methods are undisclosed."
     },
     {
       "flag": "No baseline testing framework comparison",
       "detail": "MT-LAPR is evaluated in isolation without comparison to any existing robustness testing approach (e.g., random code perturbation, existing mutation testing tools). It is unclear how much of the instability detected is attributable to the specific MR design vs. any perturbation."
     },
     {
       "flag": "Temperature=0 does not reflect practical deployment",
       "detail": "All experiments use temperature=0 for deterministic outputs, but in practice developers use higher temperatures. The robustness results may differ substantially under realistic sampling settings."
     },
     {
       "flag": "Selection bias in base samples",
       "detail": "Only samples that LLMs can successfully repair are included as base samples. This excludes harder bugs and may overestimate the practical impact of perturbations by focusing on the easiest cases."
     }
   ],
   "cited_papers": [
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Extensively explored 9 state-of-the-art LLMs for automated program repair, demonstrating LLMs significantly outperform existing APR techniques."
     },
     {
       "title": "On the robustness of code generation techniques: An empirical study on github copilot",
       "authors": ["A. Mastropaolo", "L. Pascarella", "E. Guglielmi"],
       "year": 2023,
       "relevance": "Studied robustness of GitHub Copilot in code generation, finding ~46% of cases produce different recommendations when prompts are modified."
     },
     {
       "title": "NLPerturbator: Studying the robustness of code LLMs to natural language variations",
       "authors": ["J. Chen", "Z. Li", "X. Hu", "X. Xia"],
       "year": 2024,
       "arxiv_id": "2406.19783",
       "relevance": "Found that perturbed prompts can decrease code generation performance by up to 21.2% across six code LLMs."
     },
     {
       "title": "Copiloting the copilots: Fusing large language models with completion engines for automated program repair",
       "authors": ["Y. Wei", "C. S. Xia", "L. Zhang"],
       "year": 2023,
       "relevance": "Proposes hybrid LLM-based program repair combining LLMs with code completion engines."
     },
     {
       "title": "RepairAgent: An autonomous, LLM-based agent for program repair",
       "authors": ["I. Bouzenia", "P. Devanbu", "M. Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Proposes an autonomous LLM-based agent for program repair, directly relevant to agentic AI workflows for code."
     },
     {
       "title": "Syntactic robustness for LLM-based code generation",
       "authors": ["L. Sarker", "M. Downing", "A. Desai", "T. Bultan"],
       "year": 2024,
       "arxiv_id": "2404.01535",
       "relevance": "Studies syntactic robustness of LLM-based code generation, directly related to code perturbation effects on LLMs."
     },
     {
       "title": "The plastic surgery hypothesis in the era of large language models",
       "authors": ["C. S. Xia", "Y. Ding", "L. Zhang"],
       "year": 2023,
       "relevance": "Investigates LLMs' APR capabilities in the context of the plastic surgery hypothesis about code reuse for patches."
     },
     {
       "title": "Hybrid automated program repair by combining large language models and program analysis",
       "authors": ["F. Li", "J. Jiang", "J. Sun", "H. Zhang"],
       "year": 2024,
       "arxiv_id": "2406.00992",
       "relevance": "Proposes combining LLMs with program analysis for APR, relevant to understanding LAPR technique design."
     },
     {
       "title": "VulRepair: a T5-based automated software vulnerability repair",
       "authors": ["M. Fu", "C. Tantithamthavorn", "T. Le", "V. Nguyen", "D. Phung"],
       "year": 2022,
       "relevance": "Uses T5 models for automated vulnerability repair, relevant to LLM-based code repair and the CodeT5 model used in this paper."
     },
     {
       "title": "PromptBench: Towards evaluating the robustness of large language models on adversarial prompts",
       "authors": ["K. Zhu", "J. Wang", "J. Zhou"],
       "year": 2023,
       "arxiv_id": "2306.04528",
       "relevance": "Benchmark for evaluating LLM robustness to adversarial prompts, foundational work on prompt sensitivity that motivates this paper."
     },
     {
       "title": "Exploring and unleashing the power of large language models in automated code translation",
       "authors": ["Z. Yang", "F. Liu", "Z. Yu"],
       "year": 2024,
       "relevance": "Studies LLMs for automated code translation, relevant to understanding LLM capabilities and limitations in code tasks."
     },
     {
       "title": "How effective are neural networks for fixing security vulnerabilities",
       "authors": ["Y. Wu", "N. Jiang", "H. V. Pham"],
       "year": 2023,
       "relevance": "Evaluates neural networks for security vulnerability repair, relevant to LLM-based code repair effectiveness."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The framework and readability-improvement preprocessing concept are actionable for teams deploying LAPR tools, but no code is released for immediate use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "LLM sensitivity to input formatting is already well-known; the readability-robustness correlation is a useful quantification but not deeply surprising."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Highlights that LLM-based repair tools are unreliable under common code variations, relevant to deployment safety but not a novel attack vector."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenge to specific companies or benchmarks."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code repository, no demo, no installable tool provided."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses LLaMA3 and Mistral (moderately well-known) but not tier-1 brands like GPT-4 or Copilot."
     }
   }
 }