ai-research-survey

scan.json (30464B)

---

 {
   "paper": {
     "title": "Automated Repair of Programs from Large Language Models",
     "authors": [
       "Zhiyu Fan",
       "Xiang Gao",
       "Martin Mirchev",
       "Abhik Roychoudhury",
       "Shin Hwei Tan"
     ],
     "year": 2022,
     "venue": "arXiv preprint",
     "arxiv_id": "2205.10583",
     "doi": "10.48550/arXiv.2205.10583"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval",
     "qualitative"
   ],
   "key_findings": "Automatically generated code from Codex shares common defect patterns with human-written code, with 57% of bugs being algorithm-related and 11% syntax errors. Existing APR tools TBar and Recoder can fix only 9 and 11 incorrect solutions respectively, mostly limited to single-hunk fixes. Codex edit mode (Codex-estm) outperforms both by fixing 16 solutions and can generate patches at flexible locations beyond the given suspicious statement. Combining patch spaces of APR tools and Codex/Codex-e produces required ingredients for more complex multi-hunk fixes.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section VII states 'we will make our scripts available upon acceptance' — a promise of future release, not an actual release. No repository URL or archive link is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes the LMDefects dataset with 113 Java programming tasks but provides no download URL or archive link. Section VII states 'We also release our dataset and classification result for public verification' but no concrete link is given."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section II mentions hardware ('Ubuntu-16.04 server, with 64GB RAM and Intel(R) Xeon(R) CPU E5-2660 @ 2.00GHz, and NVIDIA Titan V GPU') but provides no software dependency specifications, requirements.txt, or library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a high level but without concrete commands or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (raw counts of patches and fixed tasks). No confidence intervals or error bars are provided in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares TBar, Recoder, and Codex-e approaches based solely on raw counts (e.g., 9 vs 11 vs 16 correct patches) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw counts of patches and fixed tasks. No formal effect sizes (Cohen's d, odds ratios, etc.) are computed. While baseline context exists through comparison, the small numbers make formal effect sizes important."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset comprises 113 tasks and 335 incorrect solutions, but no justification is given for why this sample size is adequate. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are from single experimental runs. No variance, standard deviation, or spread measures are reported across multiple runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares three repair approaches: TBar (pattern-based), Recoder (learning-based), and three variants of Codex-e (edit mode). Results are compared across all tools in Tables III, IV, V, and Figure 7."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "TBar (2019) and Recoder (2021) are described as 'the most recent representative' of different APR approaches and 'have reported the best results by generating the highest number of correct patches on the Defects4J benchmark.' Codex-e was newly released in March 2022."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The three Codex-e strategies (Codex-ebug, Codex-eline, Codex-estm) systematically vary the level of fault localization guidance, effectively ablating the contribution of location information. Table V and Section V analyze the impact."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports plausible patches (pass public tests), correct patches (pass public + private tests), and correctly fixed tasks. These measure different aspects of repair effectiveness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two annotators manually classified all 335 incorrect solutions into defect categories (Section III), constructed ground truth patches, and cross-validated results. They also manually analyzed Codex-e patches for flexible fault localization."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Public LeetCode tests guide APR tools, while private LeetCode tests serve as the held-out validation set. 'The patched solutions are then validated using (1) the public tests, and (2) the held-out (private) tests in the LeetCode platform.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables II, IV, and V provide detailed breakdowns by defect category (14 sub-categories), difficulty level (easy/medium), and per-tool results for each category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV discusses specific failure cases: Figure 5 shows a multi-hunk bug neither tool could fix due to lack of dependency awareness. Section III analyzes negative symptoms in auto-generated code (wrong algorithms, similar code blocks, irrelevant helpers)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that existing APR tools are 'still quite limited' — TBar and Recoder fail on all multi-hunk bugs (0/62). They explicitly discuss three challenges: limited search space, inability to generate multi-edit patches, and lack of program dependency awareness."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: (1) common mistakes finding is backed by Section III's defect classification; (2) Codex-e being 'similar to or better than' TBar/Recoder is supported by Tables III-V (16 vs 9 and 11 correct patches); (3) combining search spaces is supported by Table VI."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'We think that Codex-estm outperforms Recoder because: (1) Codex-estm can produce complex patches at flexible locations; and (2) Codex-estm is trained on a much larger dataset.' These are speculative causal attributions without controlled experiments isolating these factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Automated Repair of Programs from Large Language Models' is broader than the evidence, which covers only one model (Codex) on Java LeetCode problems. While the threats section notes 'our experiments may not generalize beyond the studied configurations and other programming languages beyond Java,' the title and many implications sections frame findings broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for why Codex-e outperforms APR tools — e.g., differences in compute budget (50 attempts for Codex-e vs 15-minute timeout for APR tools), or whether the improvement is simply due to having more attempts rather than better repair capability."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures 'correct patches' (passing all public and private LeetCode tests) and 'plausible patches' (passing only public tests). These are directly what is claimed — no proxy gap between measurement and claims about repair effectiveness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'code-davinci-002' for Codex and 'code-davinci-edit-001' for Codex-e (Section II). Training data cutoff (June 2021) is also provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows the actual prompt format (function signature + problem description). Codex-e instructions are given verbatim: 'Fix bug in the program', 'Fix line N', 'Fix s1'. Stop sequences are listed: 'public', 'class', '//', 'System.out.print'."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section II reports: temperature 0.8, max token length 2048, 50 candidate solutions, best-of selection for top 5, stop sequences. Codex-e: 10 suspicious statements × 5 edits = 50 attempts, temperature 0.8. APR timeout: 15 minutes."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The workflow is a simple pipeline: Codex generates solutions → test against public tests → APR tools fix failures → validate against private tests."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II documents: contests from July 2021 to April 2022, 40 weekly + 20 biweekly contests, easy and medium difficulty only, 7 tasks with customized data structures excluded, post-June-2021 temporal filter, manual conversion of public tests to JUnit format, and bracket mismatch pre-fixing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threats to Validity' provides a dedicated discussion of both external and internal validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: annotator bias (14 initial disagreements resolved through discussion), Java-only evaluation, single language model (Codex), specific configurations that may not generalize, confirmation of training data overlap with Codex-e developer, and potential bugs in automated scripts."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'our experiments may not generalize beyond the studied configurations and other programming languages beyond Java' and 'our study only evaluates on the Codex language model and the Codex edit mode. The reported findings may not generalize beyond the studied model.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No download link or archive is provided for the LMDefects dataset. The paper promises release but provides no actual access to the raw data (incorrect solutions, ground truth patches, classification labels)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section II details: LeetCode contests from July 4, 2021 to April 6, 2022, 40 weekly + 20 biweekly contests, easy and medium problems, temporal filter for contamination avoidance, resulting in 60 easy + 53 medium = 113 tasks."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is LeetCode contest problems, which is a publicly available problem set — not a recruited sample."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: crawl 60 contests → 113 tasks (60 easy, 53 medium) → generate 50 solutions each → select top 5 → 46 tasks solved, 67 unsolved → 335 incorrect solutions from unsolved tasks → manual defect classification by two annotators."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments or funding section is present in the paper. No grants, sponsors, or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: National University of Singapore, Beihang University, and Southern University of Science and Technology. No commercial affiliation with OpenAI or the Codex product being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence because no funding source is disclosed. The authors use OpenAI's Codex API but are from independent academic institutions."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section II states: 'both trained on data up to Jun 2021' for both Codex code-davinci-002 and Codex-e code-davinci-edit-001."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly addressed: 'To prevent the case where the collected dataset was used in the training set of Codex, we only consider contests that are released after Jun 2021.' Section VII also confirms with the Codex-e developer that training data is the same."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The temporal split (only contests after June 2021) directly addresses contamination risk. The paper explicitly discusses why existing datasets were avoided: 'Codex was already trained on GitHub where solutions for many previous programming tasks exist (e.g., APPS, CodeContest).'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The two annotators who classified defects are researchers performing analysis, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes auto-generated code and applies automated/manual repair tools."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs reported for Codex calls despite generating 50 solutions per task (113 tasks) and 50 edit attempts per incorrect solution for Codex-e. No tokens consumed or cost per task is mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is listed (server specs, GPU) and APR timeout is 15 minutes, but total compute budget (GPU hours, API spend, total wall-clock time) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.8 introduces stochasticity in generation but no seed sensitivity analysis is performed. Results are from a single experimental run with no variance across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 50 candidate solutions generated per task, top 5 selected. For Codex-e: 5 possible edits for each of 10 suspicious statements (50 attempts total). APR tools run once with 15-minute timeout."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Settings are adopted from prior work without reporting a search budget. The paper states temperature 0.8 has 'the best performance' citing prior work, but no search over hyperparameters is described or budgeted for this study."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All three Codex-e strategies are reported without cherry-picking. Configuration choices (temperature, sampling) are justified by citing prior work: 'We reuse the same setting as prior work.'"
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate existing tools (TBar, Recoder, Codex-e) they did not build, which mitigates self-comparison bias. However, they do not explicitly acknowledge or discuss this potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Codex-e gets 50 attempts per solution while APR tools get a 15-minute timeout. Performance is not compared at matched compute budgets, and the asymmetry in compute/attempts is not discussed as a potential confound."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether LeetCode contest problems are representative of real-world auto-generated code repair scenarios. LeetCode problems are algorithmic competition tasks, which may differ substantially from production code generation use cases."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TBar uses statistical fault localization, Recoder uses its own learned localization, and Codex-e uses three different instruction strategies. These scaffolding differences are not controlled when comparing tools — performance differences could stem from the scaffolding rather than the core repair capability."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly uses temporal split: only LeetCode contests released after June 2021 (Codex training cutoff) to prevent the model from having seen solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The study cleanly separates public tests (used for repair guidance) from private tests (used for validation). The evaluation setup is transparent about what information each tool receives."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LeetCode problems from the same contests share structural similarities, or whether 5 solutions from the same task are treated as independent observations when they come from the same prompt."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Temporal split is used as a concrete prevention method. Additionally, the authors 'confirmed with the developer of Codex-e that Codex and Codex-e use the same dataset for training,' verifying no additional training contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "Auto-generated code from Codex shares common programming mistakes with human-crafted solutions, with 57% being algorithm-related bugs and 11% syntax errors.",
       "evidence": "Section III, Table II: Manual classification of 335 incorrect solutions into 14 defect categories by two annotators. Defect overlap with Codeflaws benchmark confirmed.",
       "supported": "strong"
     },
     {
       "claim": "Existing APR tools (TBar and Recoder) can fix only a small number of bugs in auto-generated code — 9 and 11 correct patches respectively.",
       "evidence": "Section IV, Table III: TBar produces 6 easy + 3 medium correct patches (6 tasks fixed); Recoder produces 6 easy + 5 medium correct patches (8 tasks fixed). Neither tool fixes any multi-hunk bugs.",
       "supported": "strong"
     },
     {
       "claim": "Codex edit mode (Codex-estm) outperforms TBar and Recoder by producing 16 correct patches, including multi-hunk fixes.",
       "evidence": "Section V, Table V: Codex-estm fixes 14 single-hunk + 2 multi-hunk = 16 solutions total. Figure 7 Venn diagram shows TBar patches are a proper subset of Codex-estm ∪ Recoder.",
       "supported": "moderate"
     },
     {
       "claim": "The effectiveness of Codex-e with specific fault location guidance (Codex-estm, 16 fixes) is nearly comparable to its effectiveness without any location guidance (Codex-ebug, 15 fixes).",
       "evidence": "Section V, Table V: Codex-ebug fixes 15 solutions, Codex-estm fixes 16. The fixed defect categories differ — Codex-ebug fixes more multi-hunk, Codex-estm fixes more single-hunk.",
       "supported": "moderate"
     },
     {
       "claim": "Combining patch spaces of TBar and Codex-e produces required ingredients for more complex fixes (9 solutions vs 4 and 5 individually).",
       "evidence": "Section V-B, Table VI: TBar+Codex-e covers 9 incorrect solutions' patch ingredients, with 2 not coverable by either tool alone. TBar+Codex covers 12 total.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Small sample sizes for comparisons",
       "detail": "Tool comparisons are based on single-digit fix counts (e.g., TBar: 9, Recoder: 11, Codex-estm: 16). Differences of 2-7 fixes without statistical testing make it difficult to distinguish signal from noise."
     },
     {
       "flag": "No statistical testing or uncertainty quantification",
       "detail": "All comparisons between tools are based on raw counts without any significance tests, confidence intervals, or repeated runs. The stochastic nature of Codex generation (temperature 0.8) means results could vary substantially across runs."
     },
     {
       "flag": "Uncontrolled compute asymmetry",
       "detail": "Codex-e gets 50 attempts per solution (10 statements × 5 edits) while APR tools get a 15-minute timeout. The compute budget differs substantially across tools, making direct comparison of 'effectiveness' potentially misleading."
     },
     {
       "flag": "Dataset not actually released",
       "detail": "The paper proposes LMDefects as a contribution but only promises future release ('we will make our scripts available upon acceptance') without providing any URL. The dataset availability claim is unverifiable."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and the HumanEval benchmark for evaluating LLM code generation — the primary model evaluated in this study."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Y. Li", "D. Choi", "J. Chung"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Major LLM code generation system evaluated on competitive programming, compared in this study's introduction."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["D. Hendrycks", "S. Basart", "S. Kadavath"],
       "year": 2021,
       "relevance": "Proposes the APPS benchmark for evaluating code generation on programming challenges — establishes pass@k evaluation methodology used in this study."
     },
     {
       "title": "TBar: Revisiting template-based automated program repair",
       "authors": ["K. Liu", "A. Koyuncu", "D. Kim", "T. F. Bissyandé"],
       "year": 2019,
       "relevance": "State-of-the-art pattern-based APR tool used as primary baseline for fixing LLM-generated code."
     },
     {
       "title": "A syntax-guided edit decoder for neural program repair",
       "authors": ["Q. Zhu", "Z. Sun", "Y.-a. Xiao"],
       "year": 2021,
       "relevance": "Recoder — learning-based APR tool with syntax-guided decoder, used as primary baseline and best-performing DL-based repair tool on Defects4J."
     },
     {
       "title": "Automatically finding patches using genetic programming",
       "authors": ["W. Weimer", "T. Nguyen", "C. L. Goues", "S. Forrest"],
       "year": 2009,
       "relevance": "GenProg — foundational search-based APR technique that established the generate-and-validate repair paradigm."
     },
     {
       "title": "Can OpenAI Codex and other large language models help us fix security bugs?",
       "authors": ["H. Pearce", "B. Tan", "B. Ahmad", "R. Karri", "B. Dolan-Gavitt"],
       "year": 2021,
       "arxiv_id": "2112.02125",
       "relevance": "Directly relevant study on using LLMs for bug fixing, focusing on security vulnerabilities."
     },
     {
       "title": "Can OpenAI's Codex fix bugs?: An evaluation on QuixBugs",
       "authors": ["J. A. Prenner", "H. Babii", "R. Robbes"],
       "year": 2022,
       "relevance": "Closely related work evaluating Codex's ability to fix bugs, using the QuixBugs benchmark."
     },
     {
       "title": "An empirical evaluation of GitHub Copilot's code suggestions",
       "authors": ["N. Nguyen", "S. Nadi"],
       "year": 2022,
       "relevance": "Evaluates Copilot code quality on LeetCode tasks — directly comparable to this study's evaluation setup."
     },
     {
       "title": "Jigsaw: Large language models meet program synthesis",
       "authors": ["N. Jain", "S. Vaidyanath", "A. Iyer"],
       "year": 2022,
       "relevance": "Combines program synthesis with LLMs for code repair, an approach this paper recommends for future work."
     },
     {
       "title": "CoCoNuT: combining context-aware neural translation models using ensemble for program repair",
       "authors": ["T. Lutellier", "H. V. Pham", "L. Pang"],
       "year": 2020,
       "relevance": "Context-aware neural APR approach that encodes program context for patch generation."
     },
     {
       "title": "CURE: Code-aware neural machine translation for automatic program repair",
       "authors": ["N. Jiang", "T. Lutellier", "L. Tan"],
       "year": 2021,
       "relevance": "DL-based APR tool using a programming language model for repair, representative of learning-based approaches."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin", "A. Odena", "M. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Foundational study on using LLMs for program synthesis, evaluating multiple models on code generation benchmarks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The idea of using APR to fix LLM-generated code is practically relevant to anyone using Copilot, but no tool is released for direct use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that Codex-e outperforms dedicated APR tools is mildly surprising, but the general limitations of APR tools are well known."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the study focuses on code correctness for programming contest problems."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy — straightforward empirical comparison with balanced discussion of tool strengths and weaknesses."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo is released; the dataset is promised for future release but not available."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Evaluates OpenAI's Codex (the model behind GitHub Copilot), which is a well-known product."
     }
   }
 }