ai-research-survey

scan.json (29906B)

---

 {
   "paper": {
     "title": "Repairing Bugs in Python Assignments Using Large Language Models",
     "authors": [
       "Jialu Zhang",
       "José Cambronero",
       "Sumit Gulwani",
       "Vu Le",
       "Ruzica Piskac",
       "Gustavo Soares",
       "Gust Verbruggen"
     ],
     "year": 2022,
     "venue": "arXiv.org",
     "arxiv_id": "2209.14876",
     "doi": "10.48550/arXiv.2209.14876"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to MMAPR's source code is provided anywhere in the paper. The paper mentions '600 lines of Python code' but does not release it."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is derived from a third-party dataset of student programs from an Indian university (referenced as [28]), but no download link is provided for either the original dataset or their derived benchmark of 286 program pairs."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a mix of Python and open-source software libraries' and 'a Windows VM with an Intel i7 CPU and 32 GB of RAM' but provides no requirements.txt, library versions, or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. A researcher would need to re-implement the system from the paper's description."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables report standard deviations of token edit distances across programs within each assignment, but no confidence intervals or error bars are reported for the repair rates themselves. No uncertainty quantification over experimental runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MMAPR outperforms the baseline (86.71% vs 67.13%) but provides no statistical significance tests (no p-values, no t-tests, no bootstrap tests) to support the comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports repair rates with baseline context (86.71% vs 67.13%, 96.50% with few-shots) and token edit distances (31.40 vs 42.50), providing enough context to assess the magnitude of improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section VI notes 'The size of the dataset is on par with the state-of-the-art automated program repair techniques' but provides no formal justification or power analysis for 286 programs being sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Standard deviations in Tables I-IV are across programs within a single run, not across multiple independent experimental runs. With temperature=0.8, results are stochastic, but no multi-run variance is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares MMAPR against a composed baseline of BIFI (syntax repair) + Refactory (semantic repair), described in Section V as combining state-of-the-art tools for each repair type."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "BIFI (2021) and Refactory (2019) were recent state-of-the-art tools at the time of this 2022 paper. The paper explicitly describes them as 'state-of-the-art' for their respective repair types."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B presents a thorough ablation study covering program chunking (Table III), iterative querying (Table IV), few-shot learning (Table I), and multimodal prompts (Figure 7), each tested independently."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used throughout: repair rate (percentage of programs fixed) and mean token edit distance (TED) measuring patch size. Both are reported in Tables I and II."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via syntax oracles (Python parser) and semantic oracles (test suites). No human evaluation of repair quality, educational value, or student comprehension is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "For the few-shot learning setting, the paper uses other students' programs as shots but does not explicitly describe a held-out test set or separation protocol to prevent leakage between few-shot examples and evaluation programs."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables I, II, III, and IV all provide per-assignment breakdowns (15 problem IDs) showing repair rates and TED for each assignment, not just aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses where BIFI fails (Section V-A, V-B4) but does not analyze specific cases where MMAPR itself fails. The 13.29% of programs MMAPR could not repair (without few-shots) are not examined."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Codex Edit model 'did not perform as well' (Section V), that removing program chunking increases TED by 41.79% (Table III), and that iterative querying slightly increases TED while improving repair rate (Table IV)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims MMAPR 'can fix more programs and produce smaller patches on average.' Table I confirms 86.71% vs 67.13% repair rate and 31.40 vs 42.50 mean TED, directly supporting both claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'by performing iterative querying the repair rate rises from 82.87% to 86.71%' are supported by controlled ablation studies (Section V-B) that isolate each component's contribution through single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title scopes to 'Python Assignments.' Section VI explicitly states: 'We only evaluated MMAPR on Python programs' and 'We carried out our evaluation on one particular set of 286 student programs,' acknowledging the bounded scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section VI discusses generic threats (test-based validation, single dataset, Python only) but does not consider alternative explanations for the observed improvements, such as whether Codex's training data overlap with the benchmark could explain high repair rates."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures repair rate (test suite pass) and token edit distance, and explicitly acknowledges in Section VI that 'Validating program correctness through tests is not as strong as formal verification,' distinguishing the proxy from the ideal outcome."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'OpenAI's Codex' and 'the completion model' (Section V) but does not specify which Codex model version (e.g., code-davinci-002 or code-cushman-001). No API version or snapshot date is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 4, 5, and 6 provide concrete prompt examples showing the actual structure and content used for both syntax and semantic phases, including code, error messages, problem descriptions, and test cases. The prompt construction is fully specified in Section IV."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V states temperature = 0.8, K = 10 candidates per prompt, top 10 candidates selected by average token log probabilities, and maximum 2 syntax iterations."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV describes the full MMAPR pipeline in detail: program chunking (Algorithm 1), syntax phase with iterative querying, semantic phase with multimodal prompts and few-shot selection, and candidate selection by edit distance. Figure 3 provides an architecture diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section V describes the benchmark derivation: started with 18 assignments, removed 3 for specific reasons, selected students with eventually correct programs, collected the latest version with syntactic mistakes, resulting in 286 program pairs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' is a dedicated section discussing three specific limitations of the study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses threats specific to this study: (1) test-based validation is weaker than formal verification, (2) evaluation on one particular set of 286 programs from one university, (3) only evaluated on Python programs."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VI explicitly states: 'We only evaluated MMAPR on Python programs' and 'We carried out our evaluation on one particular set of 286 student programs,' clearly bounding what was not tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the original student programs nor the derived benchmark of 286 program pairs are made available. Only aggregated results are shown in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section V describes the data source: 'introductory Python assignments collected by third-party authors in a large Indian university,' referencing the Python version of dataset [28], with 18 assignments containing problem descriptions, test suites, and student authoring histories."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited in this study. The student programs come from a pre-existing educational dataset collected by third-party authors."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section V documents the filtering pipeline: 18 assignments → removed 3 (file/PDF requirements) → selected students with eventually correct programs → collected latest version with syntax mistakes → 286 program pairs. Each stage's criteria are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section or grant acknowledgments are included. The paper notes the first author was a Microsoft intern but does not disclose formal funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Jialu Zhang and Ruzica Piskac at Yale University; José Cambronero, Sumit Gulwani, Vu Le, Gustavo Soares, and Gust Verbruggen at Microsoft."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Five of seven authors are Microsoft employees. Microsoft has a significant investment in OpenAI (whose Codex is the core component). Microsoft has a financial interest in demonstrating LLM utility for developer tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is included in the paper, despite the clear Microsoft–OpenAI relationship."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state Codex's training data cutoff date. Codex was trained on GitHub code, and the student assignments could overlap with training data. No cutoff is mentioned."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Codex's training data includes similar or identical programming assignments. The student programs are from an Indian university and may have been posted online."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is derived from university assignments that may have appeared online before Codex's training cutoff. No contamination analysis is performed, despite Codex being trained on millions of GitHub repositories."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates an automated repair tool on pre-existing student program data."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Student programs are from a pre-existing dataset collected by third parties."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper analyzes student programs, not the students themselves."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time per repair are reported. The paper limits syntax iterations to 2 and candidate count to 10 but does not quantify actual cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a Windows VM with an Intel i7 CPU and 32 GB of RAM' for running experiments but does not state total API spend, total compute time, or number of Codex API calls made."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "With temperature=0.8, Codex outputs are stochastic, but no multi-seed or multi-run sensitivity analysis is reported. Results appear to be from a single execution."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many independent runs of the full pipeline produced the reported results. No mention of averaging across runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We set the temperature to 0.8 based on preliminary experiments' (Section V) but does not report how many configurations were tried or the search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.8 and K=10 were selected based on 'preliminary experiments' but no details on how many alternatives were tested or what validation data was used for selection."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implemented the BIFI+Refactory baseline pipeline themselves and compared it against their own MMAPR system. No acknowledgment that their implementation of the baseline may underperform the original authors' setup."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "BIFI generates 50 candidates while MMAPR uses multiple prompts each generating 10 candidates. The compute budgets differ but are not compared or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section VI acknowledges that 'Validating program correctness through tests is not as strong as formal verification,' explicitly questioning whether the benchmark (test-suite pass) measures true correctness."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMAPR's multi-stage scaffold (chunking, iterative querying, multimodal prompts) differs fundamentally from BIFI+Refactory's pipeline. Improvements could come from the scaffold rather than Codex, but this confound is not addressed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The student programs are from an existing dataset that may predate Codex's training data. Codex could have seen identical or very similar programming exercises. No temporal analysis is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the rich prompt information (problem descriptions, test cases) provides an unfair advantage compared to real-world deployment where such information might not be available."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Codex's training data includes the same or similar introductory programming exercises from online educational platforms."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, n-gram overlap analysis, or decontamination steps."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "claims": [
     {
       "claim": "MMAPR without few-shot learning repairs 86.71% of programs, compared to 67.13% for the BIFI+Refactory baseline.",
       "evidence": "Table I shows per-assignment and overall repair rates. MMAPR repairs 248/286 programs vs 192/286 for baseline (Section V-A).",
       "supported": "moderate"
     },
     {
       "claim": "MMAPR with few-shot learning achieves a 96.50% repair rate.",
       "evidence": "Table I shows adding test-case-based few-shot selection raises the repair rate from 86.71% to 96.50% across all 15 assignments.",
       "supported": "moderate"
     },
     {
       "claim": "MMAPR produces smaller patches (mean TED 31.40) compared to the baseline (mean TED 42.50).",
       "evidence": "Table I reports mean token edit distances. MMAPR's patches are 26.38% smaller on average (Section V-A).",
       "supported": "moderate"
     },
     {
       "claim": "MMAPR achieves 100% syntax repair rate compared to 80.07% for BIFI.",
       "evidence": "Table II shows MMAPR resolves syntax errors in all 286 programs with a mean TED of 5.46, versus BIFI's 80.07% rate and 25.07 mean TED.",
       "supported": "moderate"
     },
     {
       "claim": "Program chunking reduces average token edit distance by 41.79% in the syntax phase.",
       "evidence": "Table III compares MMAPR with and without program chunking: overall mean TED drops from 9.38 to 5.46 (Section V-B1).",
       "supported": "moderate"
     },
     {
       "claim": "Iterative querying (splitting syntax and semantic phases) raises repair rate from 82.87% to 86.71%.",
       "evidence": "Table IV compares single-round vs iterative approaches across all assignments (Section V-B2).",
       "supported": "moderate"
     },
     {
       "claim": "Ensembling multimodal prompts outperforms any single prompt structure.",
       "evidence": "Figure 7 shows individual prompt structures achieve varying fix rates, while ensembling achieves the highest rate (Section V-B3).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "MMAPR, a unified system using OpenAI Codex for both syntax and semantic repair of introductory Python assignments, repairs 86.71% of 286 real student programs (96.50% with few-shot learning), outperforming a BIFI+Refactory baseline at 67.13%. The system produces patches with smaller edit distances (31.40 vs 42.50 mean TED), preserving students' original program structure. Key design decisions — program chunking, iterative querying, multimodal prompt ensembling, and test-case-based few-shot selection — each contribute measurably to performance, as shown in ablation studies.",
   "red_flags": [
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Five of seven authors are Microsoft employees, and the paper evaluates OpenAI's Codex (Microsoft invested $10B+ in OpenAI). No competing interests statement is included, and the financial relationship is not acknowledged."
     },
     {
       "flag": "No variance across stochastic runs",
       "detail": "Codex is queried with temperature=0.8, making outputs stochastic. All results appear to come from a single execution. Without multi-run variance, it is impossible to know if the observed improvements are stable or a lucky draw."
     },
     {
       "flag": "No significance tests for comparative claims",
       "detail": "The paper claims MMAPR outperforms the baseline on repair rate (86.71% vs 67.13%) and edit distance (31.40 vs 42.50) but provides no statistical tests to establish whether these differences are significant."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Codex was trained on millions of GitHub repositories which likely include introductory programming exercises very similar to the benchmark. High repair rates could partially reflect memorization rather than generalized repair capability."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors built both MMAPR and the BIFI+Refactory baseline pipeline themselves. Their baseline implementation may not match the performance that the original BIFI and Refactory authors would achieve."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "relevance": "The Codex paper — the LLM used as MMAPR's core component, foundational to LLM-based code generation research."
     },
     {
       "title": "Break-it-fix-it: Unsupervised learning for program repair",
       "authors": ["Michihiro Yasunaga", "Percy Liang"],
       "year": 2021,
       "relevance": "BIFI is the state-of-the-art syntax repair baseline used for comparison; demonstrates neural program repair approach."
     },
     {
       "title": "GenProg: A generic method for automatic software repair",
       "authors": ["Claire Le Goues", "ThanhVu Nguyen", "Stephanie Forrest", "Westley Weimer"],
       "year": 2012,
       "relevance": "Foundational automated program repair work using genetic programming, one of the first generate-and-validate APR systems."
     },
     {
       "title": "Less training, more repairing please: Revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "arxiv_id": "2207.08281",
       "relevance": "Concurrent work framing program repair as zero-shot LLM code generation, directly relevant to LLM-based APR."
     },
     {
       "title": "Repair is nearly generation: Multilingual program repair with LLMs",
       "authors": ["Harshit Joshi", "José Cambronero", "Sumit Gulwani", "Vu Le", "Ivan Radicek", "Gust Verbruggen"],
       "year": 2022,
       "arxiv_id": "2208.11640",
       "relevance": "Closely related work from overlapping authors exploring LLM-based multilingual program repair."
     },
     {
       "title": "Refactoring based program repair applied to programming assignments",
       "authors": ["Yang Hu", "Umair Z. Ahmed", "Sergey Mechtaev", "Ben Leong", "Abhik Roychoudhury"],
       "year": 2019,
       "relevance": "Refactory is the semantic repair baseline; state-of-the-art symbolic repair for student assignments."
     },
     {
       "title": "sk_p: A neural program corrector for MOOCs",
       "authors": ["Yewen Pu", "Karthik Narasimhan", "Armando Solar-Lezama", "Regina Barzilay"],
       "year": 2016,
       "relevance": "Early neural approach to student program repair for MOOCs, relevant to understanding the evolution of neural APR in education."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B. Brown"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing few-shot learning with LLMs, foundational to the prompt-based approach used in MMAPR."
     },
     {
       "title": "A conversational paradigm for program synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroshi Hayashi"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "CodeGen model for program synthesis, mentioned as an alternative LLMC that could be used in place of Codex."
     },
     {
       "title": "Automated clustering and program repair for introductory programming assignments",
       "authors": ["Sumit Gulwani", "Ivan Radicek", "Florian Zuleger"],
       "year": 2018,
       "relevance": "Symbolic APR for education using clustering of student submissions, a key prior approach MMAPR aims to improve upon."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li"],
       "year": 2022,
       "relevance": "Demonstrates LLM code generation at competition level, relevant to understanding LLM capability in programming tasks."
     },
     {
       "title": "The robots are coming: Exploring the implications of OpenAI Codex on introductory programming",
       "authors": ["James Finnie-Ansley", "Paul Denny", "Brett A. Becker", "Andrew Luxton-Reilly", "James Prather"],
       "year": 2022,
       "relevance": "Explores impact of Codex on programming education, directly relevant to AI in education domain."
     },
     {
       "title": "Synchromesh: Reliable code generation from pre-trained language models",
       "authors": ["Gabriel Poesia", "Oleksandr Polozov", "Vu Le"],
       "year": 2022,
       "arxiv_id": "2201.11227",
       "relevance": "Constrained code generation from LLMs addressing spurious edit problems similar to those MMAPR's chunking addresses."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly applicable to educational settings for automated student feedback, though Codex API is now deprecated."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "LLMs fixing code is expected; the unified syntax+semantic approach and 96.5% repair rate are notable but not contrarian."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the application is educational support."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; straightforward system evaluation paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released; the system cannot be tried by others."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Microsoft authors using OpenAI's Codex; both are well-known brands in the AI/programming tools space."
     }
   }
 }