ai-research-survey

scan.json (31972B)

---

 {
   "paper": {
     "title": "An Extensive Study on Model Architecture and Program Representation in the Domain of Learning-based Automated Program Repair",
     "authors": [
       "Dániel Horváth",
       "Viktor Csuvik",
       "Tibor Gyimóthy",
       "László Vidács"
     ],
     "year": 2023,
     "venue": "IEEE/ACM International Workshop on Automated Program Repair (APR)",
     "doi": "10.1109/APR59189.2023.00013"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Command sequence representation outperforms text representation on Java APR datasets (30.64% vs 19.88% exact match on java-small), but this advantage does not hold on the JavaScript FixJS dataset. AST+text representation significantly underperforms all other representations, achieving below 1% exact match accuracy despite a larger model (346M vs 222M parameters). The study demonstrates that no single code representation works best across all datasets and model configurations, suggesting representation choice must be tuned per setting.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided at https://github.com/AAI-USZ/APR23-representations (footnote 1 in Section I): 'Our setup, data, and methods used are also available in a GitHub repository.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Both datasets are publicly available: the Java dataset is part of the CodeXGLUE benchmark (Tufano et al.), and FixJS is a published dataset. The paper's GitHub repository also makes their processed data available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section IV mentions Python 3.8, PyTorch, PyTorch-Lightning, and the transformers library, along with an RTX 3090 GPU. However, no specific library versions, requirements.txt, or Dockerfile are provided — just library names without version numbers."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes experimental setup (hyperparameters, datasets, hardware) in Section IV but does not provide step-by-step reproduction instructions. The GitHub repo is linked but the paper itself contains no commands, scripts, or README-style instructions for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table II reports only point estimates for accuracy and 100% accuracy. No confidence intervals, error bars, or ± notation are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'outperforms all the other representations, by at least 6%') but provides no statistical significance tests. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table III explicitly reports the differences between representations and datasets with baseline context. For example, CodeT5-base on Java small: text 19.88% vs command sequence 30.64%, difference 10.76%. Both absolute values and differences are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes are stated (58,350 Java small, 65,455 Java medium, 9,662 FixJS small, 11,410 FixJS medium) but no justification or power analysis is provided for why these sizes are sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single training runs. No standard deviations, variance across seeds, or spread measures are reported anywhere in the paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares multiple models (T5, CodeT5, RoBERTa+CodeBERT+GPTNeo) across multiple representations (text, cmdseq, AST+text) and references NSEdit as the state-of-the-art on the CodeXGLUE leaderboard (Section VI)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper references NSEdit (2022) as the state-of-the-art on CodeXGLUE at time of writing, and uses recent pre-trained models (CodeT5 2021, CodeBERT 2020). The baselines are appropriate for a 2023 workshop paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies components: pretrained vs not pretrained (T5-base vs T5-base empty), LM-head-only vs full fine-tuning (T5-base LM vs T5-base), different representations on same models, and different models on same representations. Table II shows results for all configurations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table II reports two metrics: 'Accuracy' (appears to be average token/sequence-level accuracy) and '100% accuracy' (exact match rate). Values differ substantially, e.g., T5-base on Java small text: 0.9756 Accuracy vs 0.1491 100% accuracy."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via exact match comparison with developer patches. While Section V includes qualitative examples of generated patches (Listings 1-8), there is no systematic human evaluation of output quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 clearly shows separate TRAIN, TEST, and VALID splits. The paper follows 'the standard evaluation procedure in learning-based APR approaches' (Section II) with early stopping on validation loss and evaluation on the test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table II provides results broken down by model, representation, pretrained status, dataset (Java/FixJS), and dataset size (small/medium). Table III further compares across representations and datasets."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V presents multiple failure examples with analysis: Listing 3 shows an incorrect cmdseq patch with discussion of deletion bias, Listing 7 shows an incorrect text patch with discussion of overfitting to simple patterns like 'return 1'."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: AST+text representation achieves below 1% exact match ('significantly underperform'), training only the LM head 'is not enough' (0% exact match after 42 epochs), and cmdseq underperforms text on FixJS in some configurations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims match Table II results: 19.88% on java-small with text (CodeT5-base), 11.87% on java-medium (CodeT5-base), 30.64% on java-small with cmdseq (CodeT5-base), 18.53% on medium with cmdseq (CodeT5-base). The claim that AST+text 'significantly underperform' is supported by <1% exact match results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary claims are comparative ('cmdseq outperforms text'), supported by controlled experiments that vary one factor at a time (same model, different representations). The ablation design with matched conditions is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'An Extensive Study' in 'the Domain of Learning-based Automated Program Repair' broadly, but experiments cover only 2 languages (Java, JavaScript), 2 specific datasets, and 4-5 model architectures. Conclusions like 'each deep-learning setting requires its own data representation' generalize beyond what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers brief speculation that FixJS underperformance 'may be due to' smaller dataset size and unique-sample filtering, but does not systematically discuss alternative explanations. The AST+text failure is attributed to 'insufficient model size' without rigorous analysis."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses exact match with developer patches as the evaluation metric and frames results as 'accuracy' for automated program repair. While Section I acknowledges exact match is stringent, the paper does not discuss that exact match misses semantically correct but syntactically different patches, a well-known limitation in APR evaluation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific HuggingFace model identifiers are provided: t5-base, codet5-base, codebert-base, roberta-base, gpt-neo-125M. These identify specific model checkpoints. Model parameter counts are also given (222M for base models, 346M for AST+text composite)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper fine-tunes sequence-to-sequence models rather than using prompting. Input is the source code representation fed directly to the model, not a prompt."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV reports: learning rate 5e-5, Adam optimizer with epsilon 1e-8, batch sizes (16 for small, 8 for medium), sequence lengths (256 for small, 384 for medium), max 50 epochs, early stopping with delta 0.05 and patience 8, cross-entropy loss."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are standard fine-tuned transformer models with direct input-output evaluation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes preprocessing for both datasets: Java dataset uses variable/method abstraction (e.g., myVar → VARIABLE_1) with idioms preserved, FixJS uses similar abstraction with duplicate filtering. Section II.B documents the three representation transformations (text, cmdseq, AST+text) in detail with examples in Table I."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusions (Section VII) briefly mention future directions but do not substantively discuss limitations of the current study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper acknowledges some issues in passing (e.g., 'further experiments would be needed' for AST+text, model size not optimized) but these are not framed as specific threats to the study's conclusions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It mentions that 'Optimizing the size of the model is out of the scope of this paper' but does not bound the generalizability of its claims to specific languages, datasets, or model families."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Both evaluation datasets are publicly available: the Java dataset through CodeXGLUE and FixJS through its original release. The paper's GitHub repository provides access to experimental setup and data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes both datasets: the Java dataset was mined from GitHub with normalization by Tufano et al., with sizes given. FixJS contains bug-fixing GitHub commits with abstraction and deduplication by Csuvik & Vidács. Dataset sizes and splitting are documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses standard benchmark datasets (CodeXGLUE Java code refinement, FixJS)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw datasets to model input is documented: Section III describes the original datasets, Section II.B describes how each code representation (text, cmdseq, AST+text) is generated, and Section IV describes the training/evaluation procedure."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgement section discloses multiple funding sources: ÚNKP-22-3-SZTE New National Excellence Program, European Union project RRF-2.3.1-21-2022-00004 (Artificial Intelligence National Laboratory), and project TKP2021-NVA-09 from the National Research, Development and Innovation Fund."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are affiliated with the Department of Software Engineering, University of Szeged, Hungary. Affiliations are clearly listed in the paper header. Two co-authors (Csuvik, Vidács) authored the FixJS dataset used in evaluation."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Hungarian government ministries and EU programs supporting general AI research. None of the funders have a financial stake in which code representation or model performs best."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses pre-trained models (T5, CodeT5, CodeBERT, RoBERTa) that were trained on large corpora including web text and GitHub code, but never states the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Java or FixJS test examples appeared in the pre-training data of T5, CodeT5, or CodeBERT. The Java dataset was mined from GitHub, and these models were pre-trained on GitHub code, creating a clear overlap risk."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Java dataset (Tufano et al. 2019) and CodeXGLUE benchmark (2021) were publicly available before the pre-trained models' training. This contamination risk is not discussed at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a purely computational benchmark evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported. Only training time is mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV reports hardware (RTX 3090 GPU) and training times: 'about half an hour on small, and one hour on the medium dataset' for text/cmdseq, '1 hour and 40 minutes on small, and 2 hours on medium' for AST+text on Java. FixJS training ranged from 5-20 minutes per epoch. Total training time 'from 1 hour to about 1 day.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or sensitivity analysis across seeds. All results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indication of how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper uses fixed hyperparameters (learning rate 5e-5, batch sizes, etc.) without reporting whether any search was conducted or why these values were chosen beyond citing reference [27]."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Early stopping on validation loss is used with patience 8 and minimum delta 0.05: 'we assume that the model converged, and stop the training, and load the best model so far for testing.' This is a principled model selection approach."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, making multiple comparison correction inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all model configurations and baselines themselves without acknowledging potential self-comparison bias. No reference to Lucic et al.'s finding that authors' implementations of baselines systematically underperform."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The AST+text model is 346M parameters vs 222M for the other configurations, and required halved batch sizes. Performance is compared directly without controlling for or discussing these compute differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether exact-match evaluation on CodeXGLUE/FixJS actually measures real-world APR capability. While Section I briefly mentions patch correctness issues in APR, it does not question whether these specific benchmarks have construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly as fine-tuned seq2seq systems."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The Java dataset (2019) and FixJS were publicly available before the pre-trained models were released. Models pre-trained on GitHub code may have seen solutions in the test sets."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The paper does not analyze whether the input representations leak information about the target patches."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes that 'duplicated samples can occur in the Java dataset' while FixJS filters duplicates, but does not verify independence between train and test sets or between pre-training data and evaluation data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "Command sequence representation outperforms text representation on Java datasets, achieving 30.64% exact match on java-small and 18.53% on java-medium (vs 19.88% and 11.87% for text).",
       "evidence": "Table II shows CodeT5-base with cmdseq achieving 30.64% and 18.53% on Java small/medium respectively, vs 19.88% and 11.87% with text representation. Table III confirms cmdseq advantages across most Java configurations.",
       "supported": "strong"
     },
     {
       "claim": "AST+text representation significantly underperforms compared to other representations, achieving below 1% exact match accuracy.",
       "evidence": "Table II shows the RoBERTa+CodeBERT+GPTNeo model on AST+text achieves 0.77% on Java small and 0.00% on Java medium, 0.10% on FixJS small and 0.00% on FixJS medium.",
       "supported": "strong"
     },
     {
       "claim": "The advantage of command sequence representation does not hold on the FixJS JavaScript dataset, where text representation performs better on average.",
       "evidence": "Table III shows average FixJS 100% accuracy for text is 3.34% vs 2.81% for cmdseq. The Text vs Cmd column shows negative values for some FixJS configurations (e.g., -4.96% for T5-base small, -7.54% for CodeT5-base small).",
       "supported": "strong"
     },
     {
       "claim": "Training only the language model head of a pre-trained T5 model is not sufficient for the APR task.",
       "evidence": "Table II shows T5-base LM achieves 0.00% exact match on both Java small (42 epochs) and Java medium (25 epochs), despite having the highest raw accuracy values (0.9104, 0.8577).",
       "supported": "moderate"
     },
     {
       "claim": "No single representation fits every dataset and every model — each deep-learning setting requires its own data representation.",
       "evidence": "Table III shows cmdseq is better for Java but text is sometimes better for FixJS. However, this conclusion is drawn from only 2 datasets and 3 model configurations, limiting its generalizability.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results appear to be single-run experiments with no error bars, standard deviations, or confidence intervals. Deep learning results are known to vary significantly across random seeds (Henderson et al. 2018), making single-run comparisons unreliable."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "The paper makes numerous comparative claims ('outperforms by at least 6%') without any statistical tests. With single-run results and no significance testing, observed differences may be within noise."
     },
     {
       "flag": "Uncontrolled model size confound",
       "detail": "The AST+text model has 346M parameters vs 222M for text/cmdseq models, and required halved batch sizes. The poor AST+text performance could be partially due to these training differences rather than the representation itself, but this confound is not analyzed."
     },
     {
       "flag": "Pre-training data contamination risk",
       "detail": "T5 and CodeT5 were pre-trained on large corpora including GitHub code. The Java evaluation dataset was mined from GitHub. Pre-trained models may have seen test examples during pre-training, yet no contamination analysis is performed."
     },
     {
       "flag": "Co-authors authored one of the evaluation datasets",
       "detail": "Two co-authors (Csuvik, Vidács) authored the FixJS dataset used for evaluation. While this is disclosed through the citation, it is not explicitly discussed as a potential conflict when interpreting FixJS results."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section despite making broad claims about code representation for APR. Known limitations (single runs, model size confound, limited language coverage) are not systematically discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Automatically finding patches using genetic programming",
       "authors": ["W. Weimer", "T. Nguyen", "C. Le Goues", "S. Forrest"],
       "year": 2009,
       "relevance": "Foundational APR work (GenProg) using genetic programming, establishing the template-based patch generation paradigm."
     },
     {
       "title": "Generating bug-fixes using pretrained transformers",
       "authors": ["D. Drain", "C. Wu", "A. Svyatkovskiy", "N. Sundaresan"],
       "year": 2021,
       "relevance": "DeepDebug: seminal work on pre-trained transformers for automated bug fixing, directly related to this study's approach."
     },
     {
       "title": "Exploring the limits of transfer learning with a unified text-to-text transformer",
       "authors": ["C. Raffel", "N. Shazeer", "A. Roberts"],
       "year": 2020,
       "relevance": "Introduces T5, one of the primary model architectures evaluated in this study for code repair."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Y. Wang", "W. Wang", "S. Joty", "S. C. H. Hoi"],
       "year": 2021,
       "relevance": "Introduces CodeT5, the best-performing model architecture in this study's experiments."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang"],
       "year": 2020,
       "relevance": "Pre-trained code model used as the program encoder in the AST+text configuration."
     },
     {
       "title": "Fix bugs with transformer through a neural-symbolic edit grammar",
       "authors": ["Y. Hu", "X. Shi", "Q. Zhou", "L. Pike"],
       "year": 2022,
       "relevance": "NSEdit: state-of-the-art on CodeXGLUE code refinement leaderboard using neural-symbolic edit sequences, directly compared with this study."
     },
     {
       "title": "CURE: Code-aware neural machine translation for automatic program repair",
       "authors": ["N. Jiang", "T. Lutellier", "L. Tan"],
       "year": 2021,
       "relevance": "Code-aware NMT approach to APR, demonstrating the effectiveness of leveraging code structure in neural repair."
     },
     {
       "title": "DEAR: A Novel Deep Learning-based Approach for Automated Program Repair",
       "authors": ["Y. Li", "S. Wang", "T. N. Nguyen"],
       "year": 2022,
       "relevance": "Recent deep learning approach for APR showing promising results, part of the landscape this study maps."
     },
     {
       "title": "Hoppity: Learning Graph Transformations To Detect and Fix Bugs in Programs",
       "authors": ["E. Dinella", "H. Dai", "Z. Li", "M. Naik", "L. Song", "K. Wang"],
       "year": 2020,
       "relevance": "Graph-based neural network for bug detection and fixing using AST transformations, relevant to the AST representation study."
     },
     {
       "title": "An empirical study on learning bug-fixing patches in the wild via neural machine translation",
       "authors": ["M. Tufano", "C. Watson", "G. Bavota", "M. D. Penta", "M. White", "D. Poshyvanyk"],
       "year": 2019,
       "relevance": "Source of the Java APR dataset used in this study, foundational work on NMT-based program repair."
     },
     {
       "title": "CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation",
       "authors": ["S. Lu", "D. Guo", "S. Ren"],
       "year": 2021,
       "relevance": "Benchmark suite including code refinement task used in this study's evaluation."
     },
     {
       "title": "FixJS: A Dataset of Bug-Fixing JavaScript Commits",
       "authors": ["V. Csuvik", "L. Vidács"],
       "year": 2022,
       "relevance": "JavaScript bug-fixing dataset used as one of the two evaluation datasets in this study."
     },
     {
       "title": "CODIT: Code Editing with Tree-Based Neural Models",
       "authors": ["S. Chakraborty", "Y. Ding", "M. Allamanis", "B. Ray"],
       "year": 2022,
       "doi": "10.1109/tse.2020.3020502",
       "relevance": "Tree-based neural model for code editing, related approach to this study's AST+text representation experiments."
     },
     {
       "title": "A controlled experiment of different code representations for learning-based program repair",
       "authors": ["M. Namavar", "N. Nashid", "A. Mesbah"],
       "year": 2022,
       "relevance": "Most directly related work studying code representation effects on APR, but limited to NMT models and abstraction-level variations."
     },
     {
       "title": "Studying the usage of text-to-text transfer transformer to support code-related tasks",
       "authors": ["A. Mastropaolo", "S. Scalabrino", "N. Cooper", "D. Nader Palacio", "D. Poshyvanyk", "R. Oliveto", "G. Bavota"],
       "year": 2021,
       "relevance": "Study of T5 for code-related tasks including bug fixing, demonstrating transformer effectiveness across SE tasks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Provides guidance on representation choice for APR researchers but is not a ready-to-use tool for practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "AST+text underperforming is somewhat unexpected, but overall finding that representation matters is not surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk implications in this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict — a straightforward empirical comparison study."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released on GitHub but requires training infrastructure and significant setup to reproduce."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "University of Szeged; published at a small workshop (APR 2023), not a major venue or lab."
     }
   }
 }