ai-research-survey

scan.json (32200B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "Monitor-Guided Decoding of Code LMs with Static Analysis of Repository Context",
     "authors": [
       "Lakshya A Agrawal",
       "Aditya Kanade",
       "Navin Goyal",
       "Shuvendu K. Lahiri",
       "Sriram K. Rajamani"
     ],
     "year": 2023,
     "venue": "NeurIPS 2023",
     "arxiv_id": "2306.10763"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code and data released at https://github.com/microsoft/monitors4codegen. The abstract states 'Our data and implementation are available at' this URL, and Section 1 adds 'We open source our implementation and provide an extensible Python library called multilspy.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "PRAGMATICCODE dataset is released at the same GitHub repository. Section 3 describes it as 'a publicly-released dataset of Java code repositories complete with their development environments and dependencies.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B specifies Oracle JDK 17.0.6, Apache Ant 1.10.13, Apache Maven 3.8.7, Gradle 7.3.3, CodeQL CLI 2.12.5. Section 6 specifies hardware: AMD Epyc 24-core with 220GB RAM and A100 80GB, and Intel Xeon Platinum 8168 with 290GB RAM and V100 16GB."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository is provided with the multilspy library. The paper describes the full evaluation pipeline in Section 3 and Appendices B-C with sufficient detail (model configs, hyperparameters, dataset construction). The open-source library is designed for reproducibility."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports score@k values with relative improvements but no confidence intervals or error bars. Figures 2-3 show score@k curves without error bars. All results are point estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims ('MGD outperforms X by Y%') based solely on comparing point estimates. No statistical significance tests (p-values, t-tests, bootstrap tests) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports relative improvements in parentheses for every comparison (e.g., '24.69%' for CG-350M-MGD over CG-350M on CR). Absolute baseline numbers are also provided, giving full context for the magnitude of improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset has 100 repos, 1420 methods, 10538 dereference prompts. The selection process is described but there is no justification for why these numbers are sufficient, no power analysis, and no discussion of whether the sample is large enough for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are computed via the score@k estimator (Appendix D) which aggregates across 6 samples, but no standard deviation, variance, or spread measure is reported. The paper presents single aggregate values with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each model is compared with and without MGD. Multiple models serve as baselines: CodeGen-{350M, 2B, 6B}, SantaCoder, text-davinci-003. Different prompting strategies (Standard, classExprTypes, RLPG) and decoding strategies (autoregressive, FIM) also serve as comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include SantaCoder (2023), CodeGen (2023), and text-davinci-003 (2022), all contemporary at the time of writing. RLPG (Shrivastava et al., 2022) is a recent prompt augmentation technique."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Sections 4.1-4.3 systematically ablate components: MGD alone, prompt augmentation alone, FIM alone, and their combinations. Section 4.4 analyzes by identifier complexity. Section 5 ablates across languages and coding scenarios. The microbenchmark also tests individual vs joint monitors."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: Compilation Rate (CR), Next Identifier Match (NIM), Identifier Sequence Match (ISM), and Prefix Match (PM), as defined in Section 3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is fully automated: compilation checks, identifier matching, and prefix matching. No human evaluation of generated code quality is included. The Discussion acknowledges that 'additional steps such as testing and human inspection are needed to guarantee correctness.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "PRAGMATICCODE uses repositories released after the training cutoff (March 31, 2022), creating a temporal held-out set. Since MGD requires no training, there is no risk of test-set leakage through model selection on the evaluation data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model size (Section 4.1), prompting strategy (Section 4.2), decoding strategy (Section 4.3), identifier complexity (Section 4.4/Appendix F), and coding scenario (Section 5/Appendix H). Table 1 provides per-configuration breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses limitations of static analysis (imprecise/incomplete results). The microbenchmark shows partial failures: SE2 generates an invalid dereference, VD1 generates wrong number of arguments. Section 2 notes 'If Aφ returns an empty set to start with, we abandon the current run.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G reports an 83.16% mean inference slowdown with MGD. The microbenchmark reports incomplete solutions (SE2, VD1) that require joint monitoring to fix. Section 4.1 shows that smaller models with MGD still underperform larger models on method-level metrics (ISM, PM)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims: (1) MGD 'consistently improves compilation rates and agreement with ground truth' — supported by Table 1 showing 21-25% CR improvement across all models. (2) 'SantaCoder-1.1B achieves better compilation rate and next-identifier match than text-davinci-003' — supported by Figures 2a-b. Claims are appropriately hedged."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims 'MGD improves compilation rates' — a causal claim. The study design is adequate: same model with vs without MGD on the same data constitutes controlled single-variable manipulation. Sections 4.1-4.3 systematically isolate MGD's contribution from prompting and decoding strategies."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The main evaluation is explicitly on Java (PRAGMATICCODE). The generalizability study (Section 5) is framed as a 'microbenchmark' with only 10 examples, appropriately labeled as demonstrating feasibility rather than proving broad generalization. The title references 'Code LMs' broadly but the paper is clear that main results are Java-specific."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for the improvements. For instance, it does not discuss whether the compilation rate improvement could be partially due to the monitor forcing shorter/simpler completions, or whether the identifier-level guidance has downstream effects that inflate other metrics."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The metrics (compilation rate, identifier match, prefix match) are well-aligned with the claimed contributions (type-consistent identifier generation). The paper does not overclaim — it measures type consistency and claims improvements in type consistency. Section 6 explicitly notes that 'additional steps such as testing and human inspection are needed to guarantee correctness.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Models are specified with parameter counts and origins: CodeGen-{350M, 2B, 6B}-Multi (Nijkamp et al., 2023), SantaCoder-1.1B (Allal et al., 2023), text-davinci-003 (Ouyang et al., 2022). HuggingFace Transformers implementations are used for open models, Azure API for text-davinci-003."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompts are deterministically constructed from code context. Section 3 and Appendix C fully describe the construction: Standard (local file content up to dereference, left-truncated), classExprTypes (20% budget for type definitions), FIM format. Appendix I provides the exact CodeQL query for identifying target methods. The prompt IS the code — no natural language instructions are involved."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C reports: nucleus sampling with top-p=0.95, temperatures {0.2, 0.4, 0.6, 0.8}, prompt budget 1536 tokens, generation budget 512 tokens, context window 2048 tokens. Budget allocation for classExprTypes (20%) and FIM suffix (50%) also specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. MGD is a decoding-time intervention that reshapes logits using static analysis results — there are no agents, tool use, retry logic, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B documents the full pipeline: GitHub API query (March 25, 2023) for top 1000 starred Java repos after March 31, 2022 → 731 downloaded → filtered for permissive licenses and successful CodeQL builds → 302 repos → 100 after JDT.LS initialization. Method selection uses CodeQL query (Appendix I) with complexity filters."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Discussion' contains a dedicated 'Limitations' subsection discussing that static analysis of partial programs is difficult, analyses can be imprecise and incomplete, and functional correctness is beyond scope."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations discuss threats specific to this work: 'analyzing partial and incomplete programs is still a difficult problem,' static analysis heuristics 'can be both imprecise (they can give incorrect suggestions) and incomplete (they can leave out correct suggestions),' and 'Satisfying functional-correctness specifications like pre/post-conditions and invariants is beyond the scope.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states: 'Satisfying functional-correctness specifications like pre/post-conditions and invariants is beyond the scope of this work.' Section 8 (Conclusions) states plans to 'expand the scope of MGD to more languages and deeper semantic analyses,' implicitly bounding current scope to type-consistency monitoring."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "PRAGMATICCODE dataset is released at the GitHub repo (https://github.com/microsoft/monitors4codegen), including 100 repositories with their development environments. Raw data (repos, CodeQL databases, DOTPROMPTS) is available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes the full collection procedure: GitHub API query on March 25, 2023, for top 1000 starred Java repos created after March 31, 2022. Build environment details, filtering criteria, and the CodeQL-based method identification process are all documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is public GitHub repositories with a well-defined selection process described in Appendix B."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline with counts: 1000 queried → 731 downloaded → 302 after build/license filtering → 100 after JDT.LS initialization. From these: 1420 methods identified via CodeQL, up to 20 per repo, up to 10 dereference locations per method → 10538 DOTPROMPTS instances (Appendix B)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding statement or acknowledgments section listing funding sources. The authors are all from Microsoft Research, implying corporate funding, but this is not explicitly disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors clearly list their Microsoft Research affiliations with locations (Bangalore, India and Redmond, United States) in the author block."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are from Microsoft Research. Microsoft has a direct commercial interest in code generation tools (GitHub Copilot). The research demonstrating that small models with MGD can outperform large models directly benefits Microsoft's product strategy. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Microsoft has commercial interests in code generation, and some authors may have related patents or equity, but no disclosure is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states 'the determined training dataset cutoff date (31 March 2022) of the models which we use to evaluate MGD.' Appendix B reiterates this cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses this: 'We ensure that these repositories were released publicly only after the determined training dataset cutoff date (31 March 2022) for the CodeGen, SantaCoder, and text-davinci-003 family of models' (Appendix B). This temporal filtering prevents train-test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "PRAGMATICCODE was created from repos released after the training cutoff and the benchmark itself was constructed by the authors in 2023. No prior benchmark that could appear in training data is used. The contamination issue is addressed by construction."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on the DOTPROMPTS benchmark."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates language models on code repositories."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G reports inference time comparison: CG-6B mean 22.57s vs CG-6B-MGD mean 41.34s, with 83.16% mean slowdown. Table 2 provides full statistics (mean, std, min, quartiles, max)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 describes hardware configurations: (1) AMD Epyc 24-core with 220GB RAM and Nvidia A100 80GB, (2) Intel Xeon Platinum 8168 with 290GB RAM and Nvidia Tesla V100 16GB. Azure API used for text-davinci-003. The paper explicitly states 'Our experiments do not involve any training, and we only perform inferences.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are computed from 6 samples with different temperatures (0.2, 0.4, 0.6, 0.8) but no seed sensitivity analysis is reported. No standard deviation across seeds or seed-specific results are provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 clearly states: 'we use nucleus sampling with a top-p value of 0.95 to generate n = 6 independent samples' with specified temperatures. Appendix C adds: '1 each with temperature 0.2 and 0.4, and 2 each with temperature 0.6 and 0.8.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters (temperatures, top-p, prompt budget allocations) are stated but no search budget is reported. No explanation of how these specific values were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All model configurations and their results are reported in Table 1. No cherry-picking — results are shown for every combination of model, prompting strategy, and MGD status. The paper does not select a single 'best' configuration to highlight."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all. Since no hypothesis tests are conducted, multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement MGD and evaluate it against base models. While they use original model implementations and RLPG's released code (reducing re-implementation bias), they do not acknowledge or discuss self-comparison bias. The DOTPROMPTS benchmark was also designed by the authors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Appendix G reports the 83.16% inference slowdown separately. The paper does not analyze performance as a function of compute budget, nor compare MGD vs non-MGD at matched compute budgets (e.g., could the extra time be spent generating more samples instead?)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "DOTPROMPTS evaluates type-consistent dereferences in Java. The paper does not discuss whether dereference-location code completion is a valid proxy for real-world code generation quality, or whether focusing exclusively on identifier correctness at dereference points captures meaningful code generation ability."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. MGD is a decoding-time logit masking approach, not an agentic scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Repos in PRAGMATICCODE were released after March 31, 2022, the training cutoff for all evaluated models. Appendix B: 'We ensure that these repositories were released publicly only after the determined training dataset cutoff date.'"
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "For classExprTypes, target methods are masked out to prevent leakage: 'identify the type of all expressions occurring in C (after masking out the target method to prevent leakage)' (Section 3). The prompt construction avoids including ground truth in the input."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple methods are sampled from the same repository (up to 20 per repo), and multiple dereference points per method (up to 10). The paper does not discuss whether these create non-independent test examples that could inflate result counts."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The temporal split (repos created after training cutoff) is a concrete leakage prevention method. Additionally, the paper caps methods per repo at 20 and masks target methods in prompt augmentation. These are proactive prevention measures."
       }
     }
   },
   "claims": [
     {
       "claim": "MGD consistently improves compilation rates across model sizes by 21.77%-24.69% relative improvement.",
       "evidence": "Table 1 shows CR improvements: CG-350M 52.43→65.37 (24.69%), CG-2B 57.01→70.91 (24.38%), CG-6B 58.64→72.28 (23.25%), SC 59.97→73.03 (21.77%), TD-3 62.66→74.26 (18.52%).",
       "supported": "strong"
     },
     {
       "claim": "SantaCoder-1.1B with MGD outperforms the much larger text-davinci-003 (175B) on compilation rate and next-identifier match.",
       "evidence": "Figure 2a shows SC-MGD achieving ~73% CR vs TD-3's ~63% at score@6. Figure 2b shows SC-MGD achieving ~88.4% NIM vs TD-3's ~86.2% at score@6 (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "MGD is complementary to prompt augmentation — combining both yields further improvements over either alone.",
       "evidence": "Table 1 and Section 4.2: SC-RLPG-MGD achieves 78.14% CR vs SC-RLPG 66.39% and SC-MGD 73.03%. Similar additive improvements across all metrics.",
       "supported": "strong"
     },
     {
       "claim": "MGD generalizes to multiple programming languages (Java, C#, Rust), coding scenarios, and richer semantic properties (typestates, session types).",
       "evidence": "Section 5 and Appendix H: MGDMICROBENCH with 10 examples spanning 3 languages, 4 coding scenarios, typestate and session type analyses. SC-MGD generates correct code in all scenarios.",
       "supported": "weak"
     },
     {
       "claim": "MGD improves identifier prediction most dramatically for complex (multi-subtoken) identifiers, with 21-28% relative improvement.",
       "evidence": "Appendix F and Section 4.4: For identifier complexity [4,18), CG-350M-MGD achieves parity with TD-3 and outperforms CG-6B by 11.95%. SC-MGD outperforms TD-3 by 11.53%.",
       "supported": "moderate"
     },
     {
       "claim": "MGD introduces an 83% mean inference slowdown.",
       "evidence": "Appendix G, Table 2: CG-6B mean inference 22.57s vs CG-6B-MGD 41.34s across 161 matched instances with 500 prompts.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Monitor-Guided Decoding (MGD) uses static analysis through the Language Server Protocol to reshape LLM logits at decode time, enforcing type-consistent identifier generation. On the PRAGMATICCODE Java benchmark (100 repos, 10538 prompts), MGD improves compilation rates by 21-25% across all model sizes and enables the 1.1B-parameter SantaCoder to outperform the 175B-parameter text-davinci-003 on compilation rate and identifier match. MGD is complementary to prompt augmentation and fill-in-the-middle decoding, with gains stacking. The approach generalizes to Java, C#, and Rust on a small microbenchmark, and can enforce richer semantic constraints like typestate protocols and session types.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims ('X outperforms Y by Z%') are based on point estimates without any significance tests. With 10538 data points, many comparisons, and no uncertainty quantification, it is unclear which improvements are statistically meaningful."
     },
     {
       "flag": "No variance or error bars reported",
       "detail": "Despite generating 6 samples per prompt, no standard deviation, confidence intervals, or error bars are reported on any metric. The score@k estimator aggregates results but the reader cannot assess result stability."
     },
     {
       "flag": "Tiny generalizability study",
       "detail": "The generalizability to C#, Rust, and richer semantic properties is demonstrated on only 10 hand-crafted micro-benchmark examples (MGDMICROBENCH). This is insufficient to support generalization claims beyond proof-of-concept."
     },
     {
       "flag": "Inference slowdown buried in appendix",
       "detail": "An 83% mean inference slowdown (Table 2, Appendix G) is a significant practical limitation but is not discussed in the main paper. The slowdown is only reported for CG-6B; other model configurations are not benchmarked for latency."
     },
     {
       "flag": "Corporate conflict of interest undisclosed",
       "detail": "All authors are from Microsoft Research, which has direct commercial interest in code generation tools (GitHub Copilot). The paper demonstrates that small models with static analysis can rival large models — directly relevant to Microsoft's product strategy. No conflict of interest statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark, foundational work on LLM code generation evaluation."
     },
     {
       "title": "SantaCoder: don't reach for the stars!",
       "authors": ["Loubna Ben Allal", "Raymond Li"],
       "year": 2023,
       "arxiv_id": "2301.03988",
       "relevance": "1.1B parameter code model used as primary evaluation target in this paper, demonstrates small model code generation."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang"],
       "year": 2023,
       "relevance": "Open code generation model family (350M-6B) used as baselines for evaluating MGD across parameter scales."
     },
     {
       "title": "Repository-level prompt generation for large language models of code",
       "authors": ["Disha Shrivastava", "Hugo Larochelle", "Daniel Tarlow"],
       "year": 2022,
       "arxiv_id": "2206.12839",
       "relevance": "RLPG prompt augmentation technique used as a baseline and shown to be complementary with MGD."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "doi": "10.1126/science.abq1158",
       "relevance": "Large-scale code generation system, represents encoder-decoder approach to code generation."
     },
     {
       "title": "CoCoMIC: Code Completion By Jointly Modeling In-file and Cross-file Context",
       "authors": ["Yangruibo Ding", "Zijian Wang"],
       "year": 2022,
       "arxiv_id": "2212.10007",
       "relevance": "Cross-file context for code completion using architecture modifications and additional training — contrasts with MGD's training-free approach."
     },
     {
       "title": "PICARD: Parsing Incrementally for Constrained Auto-Regressive Decoding from Language Models",
       "authors": ["Torsten Scholak", "Nathan Schucher", "Dzmitry Bahdanau"],
       "year": 2021,
       "relevance": "Constrained decoding approach for SQL that is conceptually similar to MGD, using incremental parsing for validity."
     },
     {
       "title": "Synchromesh: Reliable code generation from pre-trained language models",
       "authors": ["Gabriel Poesia", "Alex Polozov"],
       "year": 2022,
       "relevance": "Constrained decoding for code generation on SQL and DSLs; closely related approach using semantic checks during decoding."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Large-scale open code generation model, successor to SantaCoder, represents state of the art in open code LMs."
     },
     {
       "title": "Better context makes better code language models: A case study on function call argument completion",
       "authors": ["Hao Pei", "Jian Zhao"],
       "year": 2023,
       "relevance": "Uses static analysis context for function call argument completion; related approach that requires finetuning unlike MGD."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "authors": ["Fengji Zhang", "Bei Chen"],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Repository-level code completion using retrieval, complementary approach that modifies input rather than output."
     },
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models",
       "authors": ["Hao Yu", "Bo Shen"],
       "year": 2023,
       "arxiv_id": "2302.00288",
       "relevance": "Pragmatic code generation benchmark evaluating repository-level context, motivates PRAGMATICCODE dataset creation."
     },
     {
       "title": "Incoder: A generative model for code infilling and synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan"],
       "year": 2022,
       "arxiv_id": "2204.05999",
       "relevance": "Code infilling model supporting fill-in-the-middle, relevant to FIM decoding strategy evaluated with MGD."
     }
   ]
 }