ai-research-survey

scan.json (30406B)

---

 {
   "paper": {
     "title": "Towards Effectively Leveraging Execution Traces for Program Repair with Code LLMs",
     "authors": [
       "Mirazul Haque",
       "Petr Babkin",
       "Farima Farmahinifarahani",
       "Manuela Veloso"
     ],
     "year": 2025,
     "venue": "Proceedings of the 4th International Workshop on Knowledge-Augmented Methods for Natural Language Processing",
     "arxiv_id": "2505.04441",
     "doi": "10.18653/v1/2025.knowledgenlp-1.17"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Simply incorporating execution traces into LLM prompts for automated program repair provides inconsistent benefits, improving performance in only 2 of 6 model-dataset configurations. Longer and more complex traces tend to reduce repair effectiveness. LLM-optimized (summarized) traces provide the most consistent improvements across configurations. Probing studies show LLMs have limited ability to generate execution traces from scratch (15–50% exact match), suggesting real traces provide information not easily inferred by the model.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper. The authors mention implementing a fine-tuning pipeline and custom wrappers but do not release them."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All three evaluation datasets are publicly available: Refactory (Hu et al., 2019), RunBugRun (Prenner and Robbes, 2023), and HumanEval-Java (Jiang et al., 2023). The authors did not create proprietary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. The paper mentions using PySnooper and deepseek-coder-1.3b-instruct but does not list library versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. A researcher would need to reverse-engineer the experimental setup from the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates for CFA and CPA. No confidence intervals, error bars, or ± notation is provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. All comparisons between prompt types and models are based solely on comparing raw accuracy numbers without any p-values or formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Raw accuracy values (CFA, CPA) are reported for all conditions with baselines, allowing readers to assess magnitude. For example, GPT-4 on HumanEval-Java shows Trace Prompt CPA=0.713 vs Error Prompt CPA=0.662, providing sufficient context to gauge improvement magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for sample sizes. The paper uses 138 programs from Refactory (which has ~2000), 157 from HumanEval-Java, and 1000 sampled from RunBugRun, but does not explain why these particular sizes were chosen or provide power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or any spread measure is reported. Results are single-run numbers with one prediction per prompt. No indication of result stability across different conditions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are included: Error Prompts (failing test case without trace) and Self-Debug (LLM-generated traces inspired by Chen et al., 2023). Both are evaluated across all datasets and models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Self-Debug (Chen et al., 2023) and the prompt template from Xia et al. (2023) are recent. TraceFixer (Bouzenia et al., 2023) is used as a fine-tuning comparison. These are contemporary to the work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple trace format variations are tested: raw traces, collated traces, LLM-optimized traces, confidence-based selection, and trace-length-based routing. This effectively ablates the trace representation component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: Correct Fix Accuracy (CFA, percentage of fixes passing all test cases) and Correct Program Accuracy (CPA, percentage of programs with at least one correct fix)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All evaluation is automated through test-suite pass/fail. The probing studies in Section 5.2 involve manual review of diffs but this evaluates trace understanding, not the repair quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For prompting experiments, the full datasets serve as test data (no training involved). For the fine-tuning experiment (Section 5.1), an explicit 80/20 train/test split is used: '80% of the problems are randomly selected for training, and the rest are reserved for testing.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (Refactory, HumanEval-Java, RunBugRun), by model (GPT-3.5, GPT-4), and by prompt type in Tables 1 and 2. Figure 2 provides per-dataset trace complexity analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 3.3) analyzes why trace-based prompts fail, showing distributions of trace complexity for correct vs incorrect fixes. Section 5.2 qualitatively analyzes LLM trace manipulation failures (missed variable modifications in loops, wrong function returns, hallucinated exceptions)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The core finding is negative: 'Trace prompts do not consistently outperform Error Prompts on program repair' (RQ1 Summary). Collated traces are described as 'disappointingly' not improving over standard traces. Multiple configurations show traces hurting performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-matched to results. The claim of 'limited performance improvement over trace-free baselines, in only 2 out of 6 tested dataset / model configurations' is directly supported by Table 1. Claims about LLM-optimized traces being more consistent are supported by Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claims come from controlled single-variable comparisons (same programs, same model, different prompt type). The trace complexity analysis (RQ2) is correlational but is presented as an observation rather than a causal claim. The paper uses appropriate hedging ('could highlight', 'implies')."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Program Repair with Code LLMs' is broader than what is tested. Only GPT-3.5 and GPT-4 are evaluated (no open-source models). Only algorithmic/self-contained programs are used. The paper acknowledges the model limitation but the title and framing imply broader applicability to 'Code LLMs' generally."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple alternative explanations are discussed: training data format familiarity for collated traces, attention dilution from long traces, truncation effects, LLM hallucination in Self-Debug, and the qualitative generational gap between GPT-3.5 and GPT-4. Section 4.2 discusses why collated traces fail (lack of training exposure, loop stretching, truncation)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures test-suite pass rates (CFA, CPA) and frames these as measures of program repair effectiveness. The claims match the granularity of measurements — they do not overclaim that test-passing equates to broader code quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-3.5 Turbo' and 'GPT-4' without specific version identifiers (e.g., gpt-3.5-turbo-0613, gpt-4-0613). For OPT traces they mention 'GPT4-32k' without a version. For fine-tuning, 'deepseek-coder-1.3b-instruct' is specified but this is a model family name, not a snapshot."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described in natural language ('We follow the instruction template for complete function generation used by Xia et al., 2023') but the actual prompt text is not provided. Figure 1 shows example data (buggy program, test case, trace) but not the instruction template itself."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API hyperparameters (temperature, top-p, max tokens) are reported for GPT-3.5 or GPT-4. For fine-tuning, the paper says 'We use the training settings and parameters suggested by deepseek-coder developers' without listing specific values. Prompt truncation at 200 lines is the only parameter mentioned."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is single-turn prompting of LLMs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Trace generation via PySnooper is described. Postprocessing steps are stated: 'removal of timestamps and stripping of terminal formatting command sequences.' Prompt truncation at 200 lines is documented. RunBugRun wrapper for I/O handling is described. Truncation rates are reported (5% for Refactory, ~10% for RunBugRun)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion briefly mentions scope limitations but does not constitute substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper acknowledges using only GPT models and algorithmic datasets but does not frame these as specific threats or analyze their impact on conclusions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper notes 'there is undoubtedly scope for including more proprietary as well as open source models,' it does not explicitly state what the results do NOT show or what populations/settings are excluded. No dedicated scope boundary statements are made."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, generated fixes, execution traces) is made available for verification. Only aggregated accuracy numbers are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes dataset selection: 15 datasets surveyed with criteria (size, diversity, unit test availability, origin). Three datasets selected with clear rationale. RunBugRun sampling of 1000 Python bugs is described. Trace generation via PySnooper is detailed."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (Refactory, RunBugRun, HumanEval-Java)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw datasets to final experimental samples has gaps. Refactory has ~2000 faulty programs but only 138 are used without explanation of the filtering. The total number of prompts per dataset is given but intermediate steps are not fully documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The disclaimer section states the paper was prepared by JP Morgan AI Research but does not constitute a funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are clearly identified as J.P. Morgan AI Research, with locations (New York and Palo Alto) specified. The affiliation is prominent in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "J.P. Morgan does not have a direct financial stake in whether execution traces improve GPT-based program repair. The paper evaluates OpenAI models, not JP Morgan products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. The disclaimer is a legal boilerplate, not a conflicts-of-interest declaration."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-3.5 Turbo or GPT-4. This is critical since the benchmarks predate these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems appeared in GPT training data. HumanEval (published 2021), Refactory (2019), and CodeNet/RunBugRun (2021-2023) all predate or overlap with GPT model training periods."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval is a widely-known benchmark published in 2021, well before GPT-3.5 and GPT-4 training. Refactory and CodeNet are similarly public. No contamination analysis or acknowledgment is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time reported. The approach makes many GPT-3.5/GPT-4 API calls (hundreds per dataset-model pair) and OPT traces require an additional GPT-4-32k call per example, but costs are never quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. Neither API spend for prompting experiments nor GPU hours for the deepseek-coder fine-tuning are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple-seed experiments are conducted. Single predictions per prompt are generated with no analysis of sensitivity to random seeds or sampling variation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'we generate a single prediction per test case-specific prompt and aggregate across prompts when computing metrics' (Section 3.1, Metrics)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described for the prompting experiments. For the trace-length routing threshold, 6 values are tested (25-50) but no search budget is stated. Fine-tuning uses default deepseek-coder parameters without search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "For trace-length-based routing (TRL OPT), the paper reports 'we only report the best results in the table' but provides full results for all threshold values in Appendix Figures 6 and 7, enabling verification."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, despite comparing many prompt types across multiple datasets and models. No multiple comparison correction could be applied since no tests were run."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors re-implement TraceFixer's fine-tuning pipeline ('As we didn't have access to TraceFixer's code, we implemented our own fine-tuning pipeline') and Self-Debug baseline, but do not acknowledge or discuss the bias of evaluating their own implementations of baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "OPT traces require an additional GPT-4-32k call per example, and confidence-based selection requires extra LLM queries. These compute differences relative to simpler baselines are not discussed or quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses dataset selection criteria and explicitly acknowledges the gap between algorithmic benchmarks and realistic programs: 'While realistic datasets are ideal, evaluating them requires significant manual effort due to complex dependencies. Algorithmic datasets offer advantages like manageable length and easily testable, self-contained functions.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is single-turn prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. HumanEval (2021), Refactory (2019), and CodeNet (2021) all predate GPT-3.5 and GPT-4 training, meaning solutions may exist in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information. The execution traces themselves come from running the buggy program, which could provide information not available in realistic debugging scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between train and test data. The fine-tuning experiment uses a random 80/20 split but does not verify that problems don't share structural similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used (no canary strings, membership inference tests, n-gram overlap analysis, or temporal splits)."
       }
     }
   },
   "claims": [
     {
       "claim": "Simply incorporating execution traces into LLM prompts provides limited improvement over trace-free baselines, helping in only 2 of 6 model-dataset configurations.",
       "evidence": "Table 1 shows Trace Prompts outperform Error Prompts only for GPT-4 on HumanEval-Java (CPA 0.713 vs 0.662) and RunBugRun (CFA 0.558 vs 0.529). In all GPT-3.5 configurations and GPT-4 on Refactory, traces underperform or match error-only prompts.",
       "supported": "strong"
     },
     {
       "claim": "The effectiveness of trace-based prompts decreases as trace complexity (length and number of variable modifications) increases.",
       "evidence": "Figure 2 shows median trace length and variable changes are significantly higher for failing fixes than correct ones on HumanEval-Java and RunBugRun with GPT-4. Refactory shows a partial exception where trace length is lower for failing fixes.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-optimized (summarized) traces provide the most consistent performance gains across all configurations.",
       "evidence": "Table 2 shows OPT Trace achieves top-3 CPA in all 6 configurations and best or second-best CFA in all 6. However, OPT still underperforms 'RQ1 Best' (the best non-optimized prompt) in 4 of 6 CPA comparisons.",
       "supported": "moderate"
     },
     {
       "claim": "Trace-based prompting outperforms fine-tuning a smaller LLM (deepseek-coder-1.3b) for APR.",
       "evidence": "Figure 4 shows all prompting techniques outperform fine-tuned models on CPA and CFA across datasets. However, the comparison is between GPT-3.5/4 (much larger models) and deepseek-coder-1.3b with limited training data (459-517 samples).",
       "supported": "weak"
     },
     {
       "claim": "LLMs have limited ability to generate execution traces from scratch, supporting the value of real execution traces.",
       "evidence": "Table 3 shows GPT-4 trace prediction accuracy of 50% on reference Refactory programs, 26% on failing programs, and 15% on Geeks-for-geeks. Trace collating accuracy ranges from 45% to 88%.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical testing",
       "detail": "All comparisons across 6+ configurations and multiple prompt types are based solely on point estimates. No significance tests, confidence intervals, or variance measures are reported, making it impossible to assess whether observed differences are meaningful or due to chance."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "HumanEval (2021), Refactory (2019), and CodeNet (2021) were all publicly available before GPT-3.5 and GPT-4 training. The models may have seen benchmark solutions, which could differentially affect trace-based vs trace-free prompts. This is never discussed."
     },
     {
       "flag": "Unfair fine-tuning comparison",
       "detail": "The fine-tuning comparison pits a 1.3B parameter model trained on 459-517 samples against GPT-3.5/GPT-4 (orders of magnitude larger). The authors acknowledge limited training data but still present the comparison as evidence that prompting outperforms fine-tuning."
     },
     {
       "flag": "Missing model versions and hyperparameters",
       "detail": "No specific GPT model versions, API snapshot dates, or sampling parameters (temperature, top-p) are reported. GPT model behavior changes across versions, making these experiments unreproducible."
     },
     {
       "flag": "Unexplained dataset subsetting",
       "detail": "Refactory has ~2000 faulty programs but only 138 are used. No explanation is given for this 93% reduction. The selection criteria could significantly affect results."
     }
   ],
   "cited_papers": [
     {
       "title": "Teaching large language models to self-debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli", "Denny Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Key baseline: Self-Debug approach that prompts LLMs to generate their own execution traces for code repair."
     },
     {
       "title": "TraceFixer: Execution trace-driven program repair",
       "authors": ["Islem Bouzenia", "Yangruibo Ding", "Kexin Pei", "Baishakhi Ray", "Michael Pradel"],
       "year": 2023,
       "relevance": "Fine-tuning baseline that trains CodeT5 with execution traces for APR, showing 13% improvement on synthetic bugs."
     },
     {
       "title": "TRACED: Execution-aware pre-training for source code",
       "authors": ["Yangruibo Ding", "Ben Steenhoek", "Kexin Pei", "Gail Kaiser", "Wei Le", "Baishakhi Ray"],
       "year": 2023,
       "relevance": "Pre-training approach using execution traces that improved clone detection and vulnerability detection over AST-based methods."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Provides the instruction template for complete function generation used as the prompting framework in this study."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "Zero-shot LLM-based APR approach that established baselines for prompt-based program repair."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "relevance": "Evaluates code LLMs for APR and provides the HumanEval-Java dataset with synthetic bugs used in this study."
     },
     {
       "title": "SelfAPR: Self-supervised program repair with test execution diagnostics",
       "authors": ["He Ye", "Matias Martinez", "Xiapu Luo", "Tao Zhang", "Martin Monperrus"],
       "year": 2022,
       "relevance": "Uses compiler and test diagnostics during self-supervised training for APR, a precursor to trace-augmented repair."
     },
     {
       "title": "Is ChatGPT the ultimate programming assistant – how far is it?",
       "authors": ["Haoye Tian", "Weiqi Lu", "Tsz On Li", "Xunzhu Tang", "Shing-Chi Cheung", "Jacques Klein", "Tegawendé F. Bissyandé"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT on programming tasks including APR, relevant to understanding LLM capability for code repair."
     },
     {
       "title": "RunBugRun – an executable dataset for automated program repair",
       "authors": ["Julian Aron Prenner", "Romain Robbes"],
       "year": 2023,
       "relevance": "One of the three main evaluation datasets, containing a quarter million submissions for 4000 distinct problems."
     },
     {
       "title": "Can large language models reason about program invariants?",
       "authors": ["Charles Sutton", "David Bieber", "Kensen Shi", "Kexin Pei", "Pengcheng Yin"],
       "year": 2023,
       "relevance": "Examines LLM reasoning about program execution behavior, relevant to understanding trace comprehension capabilities."
     },
     {
       "title": "Code execution with pre-trained language models",
       "authors": ["Chenxiao Liu", "Shuai Lu", "Weizhu Chen", "Daxin Jiang", "Alexey Svyatkovskiy", "Shengyu Fu", "Neel Sundaresan", "Nan Duan"],
       "year": 2023,
       "relevance": "Program state prediction pre-training that improved code search and generation, related to execution-aware code models."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The idea of augmenting APR prompts with traces is interesting but results are mixed and no tools or code are released for practitioners to use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that execution traces often don't help and can hurt LLM-based repair is mildly contrarian to the intuition that more information should help."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications in this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "J.P. Morgan AI Research is a recognized corporate lab; GPT-3.5/GPT-4 are well-known models."
     }
   }
 }