ai-research-survey

scan.json (34735B)

---

 {
   "paper": {
     "title": "Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks",
     "authors": [
       "Linyuan Gong",
       "Sida Wang",
       "Mostafa Elhoushi",
       "Alvin Cheung"
     ],
     "year": 2024,
     "venue": "International Conference on Machine Learning (ICML)",
     "arxiv_id": "2403.04814",
     "doi": "10.48550/arXiv.2403.04814"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "SAFIM, a 17,720-example syntax-aware fill-in-the-middle benchmark across four languages, reveals that FIM pretraining enhances both FIM and left-to-right inference, and that pretraining methods and data quality matter more than model size. Syntax-aware truncation in post-processing dramatically improves fair comparison across model types, boosting Pass@1 for non-FIM models like CodeLLaMa-13B from 16.4% to 41.4%. DeepSeekCoder-33B leads at 69.0% average Pass@1, while 1.3B DeepSeekCoder (52.6%) outperforms GPT-3.5 (40.9%). Data contamination analysis on a temporally separated test set shows negligible impact on results.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The evaluation toolkit and dataset are available at https://github.com/gonglinyuan/safim, and the leaderboard is available at https://safimbenchmark.com.' A concrete repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released alongside the evaluation toolkit at the GitHub URL. The benchmark contains 17,720 examples from Codeforces and GitHub, publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'the Huggingface transformers library' and 'OpenAI API' but does not provide library versions, requirements.txt, Dockerfile, or a detailed environment setup section. Specific model IDs are listed in Table 7 but framework/dependency versions are absent from the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper provides detailed methodology descriptions and releases code/data, there are no step-by-step reproduction instructions in the paper itself (no 'Reproducing Results' section or commands to run)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (Tables 2-4, 8-17) report only point estimates of Pass@1 percentages with no confidence intervals or error bars. The paper justifies this by claiming the dataset's large size (17,720 examples) makes uncertainty quantification unnecessary."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported anywhere. Claims like 'StarCoder outperforms CodeGen-16B' and 'FIM pretraining enhances L2R performance' are based on comparing raw Pass@1 numbers without any statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports absolute scores with baseline context, e.g., 'the weakest CodeLLaMa model surpasses the strongest CodeGen model by 14 points' and 'StarCoder, with 15.5B parameters, achieves an average Pass@1 of 55.5%, comparable to GPT-4's 53.3%.' Per-model scores are provided for all comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 explicitly justifies the dataset size: 'Our large dataset size of 17,720 examples enables robust evaluations without the need for multiple generations and averaging, as seen in smaller datasets like HumanEval (164 programs).'"
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly generates only one completion per model per example ('we only generate one completion for each LLM on each example') and reports no variance across runs, seeds, or sampling stochasticity."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 15+ models spanning multiple families (CodeGen, InCoder, CodeLLaMa, StarCoder, DeepSeekCoder, GPT-3.5, GPT-4) as baselines against each other. Tables 4 and 15 present comprehensive comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models for the submission period: DeepSeekCoder (2024), GPT-4 (2023), CodeLLaMa (2023), StarCoder (2023), plus additional models like Mixtral, WizardCoder, and Magicoder in the appendix."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 (and Tables 11-13) provides an ablation study on the syntax-aware truncation algorithm, comparing performance with and without truncation. Tables 2/8-10 compare five prompt designs per model. Within-family size comparisons serve as implicit ablations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Pass@1 as the primary metric and CErr% (compilation/syntax error percentage) as a secondary metric in Tables 3, 11-13. Results are also broken down by task category and language."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated: execution-based testing (98.25% of examples) and syntactical match evaluation for API function calls. No human evaluation of outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is constructed from code created between April 2022 and January 2023, deliberately chosen to be after most models' training data cutoffs. Appendix A.9 further creates a separate test set from April 2023 to January 2024 to test contamination effects."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 breaks down results by three task categories (algorithmic block, control-flow, API function call). Table 16 (Appendix A.7) provides per-programming-language breakdowns across C++, Java, Python, and C#."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.8 provides a detailed case study comparing InCoder-6B's successful completion and CodeGen-16B's failure on the same task. Compilation error rates are analyzed in Tables 3/11-13. The paper discusses models producing 'extraneous content' and 'infinite output generation.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: finetuning on distilled data hurts FIM performance compared to base models (Appendix A.6), CodeLLaMa-34B underperforms the smaller 13B variant due to lack of FIM (Section 6.1), IPF prompt causes placeholder token output errors (Section 4.1), Phi models underperform relative to their HumanEval scores (Appendix A.6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: FIM pretraining enhancing FIM and L2R inference is shown in Table 2 (Section 6.1), pretraining methods mattering more than model size is demonstrated in Table 4/Figure 3 (Section 6.3), SAFIM providing fair comparisons is evidenced by the truncation ablation (Table 3) and prompt comparison (Table 2)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'FIM pretraining not only enhances FIM proficiency but also improves Left-to-Right (L2R) inference' — a causal claim based on observational comparison across model families with different architectures, data, and training procedures. While the paper acknowledges in Section 7 that 'our conclusions are drawn from comparisons across various model families trained with different paradigms, rather than from controlled experiments,' the abstract presents these causal claims without caveats."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title scopes to 'Syntax-Aware Code Fill-in-the-Middle Tasks.' Claims use hedging language ('our findings suggest'). The paper acknowledges cross-family comparisons are not controlled experiments (Section 7) and that results may vary by programming language (Appendix A.7). Scope is mostly bounded to SAFIM's three task types and four languages."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple alternative explanations: data contamination (Appendix A.9 with a dedicated experiment), differences in pretraining environments confounding cross-family comparisons (Section 1, Section 7), programming language coding style affecting results (Appendix A.7), and training data cutoff date overlap (Table 1)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@1 on execution-based tests and syntactical matching, and frames results in terms of 'code completion proficiency' and 'FIM performance.' The measurements match the granularity of claims — no broader framing of 'developer productivity' or similar proxy gaps exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 (Appendix A.3) provides exact model identifiers for every model: 'gpt-3.5-turbo-0301', 'gpt-4-1106-preview', 'Salesforce/codegen-350M-multi', 'codellama/CodeLlama-7b-hf', 'deepseek-ai/deepseek-coder-33b-base', etc."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides detailed prompt structures with actual examples for all five prompt types (L2R, PSM, SPM, IPF, 1S), including sentinel token mappings for each model family. The prompt designs are described precisely enough to reconstruct."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states: 'we use top-p random sampling with p = 0.95 and a temperature of 0.2.' The decoding strategy follows 'established practices in Fried et al. (2023).'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates raw model inference (single-turn generation) on code completion tasks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides detailed preprocessing: Codeforces solutions are filtered by unit test execution within 50% time limit, length (under 2x shortest solution), deduplication (CodeBLEU > 0.9 threshold). GitHub files are filtered by API library usage, star count (>10), and presence of comments. Section 3.2 describes AST-based task generation and validation filtering for each task category."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 contains substantive limitations discussion: 'We acknowledge a key limitation in our study: our conclusions are drawn from comparisons across various model families trained with different paradigms, rather than from controlled experiments altering pretraining paradigms within the same model.' An Impact Statement section also discusses broader concerns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: data contamination from training data overlap (Table 1, Appendix A.9 with a dedicated experiment), non-controlled cross-family comparisons (Section 7), programming language coding style affecting results (Appendix A.7), and prompt sensitivity affecting model rankings (Section 6.1)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what the results do NOT show: 'our conclusions are drawn from comparisons across various model families trained with different paradigms, rather than from controlled experiments' and proposes 'future work in pretraining such models under the same environment to validate these observations further.' The paper also scopes to decoder-only models, excluding encoder-decoder models (Section 2)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset and evaluation toolkit are publicly available at https://github.com/gonglinyuan/safim. Individual examples, unit tests, and evaluation results can be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes collection from Codeforces (problems, unit tests, solutions) and GitHub (git commits from GH Archive), with time period (April 1, 2022 to January 1, 2023), inclusion criteria (GitHub repos with 10+ stars, files with natural language comments), and four programming languages."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is sourced from public code repositories (Codeforces and GitHub) via standard scraping."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented in detail: Sections 3.1-3.2 describe source corpus collection → unit test re-evaluation → length and duplicate filtering → AST parsing → task generation → validation filtering. Numbers are provided at key stages: 490 coding questions, 8,590 unique code solutions (Codeforces), 11,936 code files (GitHub), 17,720 total examples. Per-category statistics in Table 5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section lists: 'gift from Meta, the U.S. National Science Foundation through grants IIS-1955488, IIS-2027575, ARO W911NF2110339, ONR N00014-21-1-2724, and DOE awards DE-SC0016260, DE-SC0021982.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Linyuan Gong and Alvin Cheung at University of California at Berkeley, Sida Wang and Mostafa Elhoushi at 'AI at Meta.' Two authors are at Meta, whose CodeLLaMa model is evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta provided a gift fund and two authors are Meta employees. Meta develops CodeLLaMa, one of the evaluated models. While CodeLLaMa does not consistently win (DeepSeekCoder outperforms it), the funder has a financial interest in LLM evaluation outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Two Meta-affiliated authors evaluate Meta's CodeLLaMa, but no explicit conflict-of-interest disclosure is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides training data cutoff dates for all evaluated models: GPT-3.5/GPT-4 (Sept 2021), CodeGen (Oct 2021), InCoder (≤Mar 2022), CodeLLaMa (Jul 2022), StarCoder (Mar 2022), DeepSeekCoder (Feb 2023)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensively discussed. Table 1 highlights models with date-range overlap in red. Section 3.1 explains the temporal cutoff strategy. Appendix A.9 provides a detailed contamination analysis with a new test set from April 2023 - January 2024 that has no overlap with any model."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.9 creates an entirely new dataset from a later time period (April 2023 - January 2024) with no date overlap with any model's training data. Table 17 and Figure 5 show that performance remains stable, demonstrating 'negligible impact' of potential contamination. The benchmark itself is sourced from code after April 2022 to avoid overlap with major pretraining datasets."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLMs on code completion tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on code completion using automated execution-based testing."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available code from Codeforces and GitHub."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Inclusion/exclusion criteria are for code examples, not people."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving randomized assignment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Evaluation is fully automated."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. No attrition concept applies."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, latency, tokens consumed, or API costs are reported despite evaluating 15+ models across 17,720 examples each using both local GPU inference and OpenAI API calls."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, or hardware specifications are reported. The acknowledgements thank individuals for 'providing computational resources' but do not quantify the compute budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only one completion is generated per model per example using stochastic sampling (top-p 0.95, temperature 0.2). No multi-seed analysis is conducted. The paper argues dataset size compensates but does not test seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 explicitly states: 'we only generate one completion for each LLM on each example and report the percentage of first-attempt passes, i.e., Pass@1, as our evaluation metric.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature (0.2) and top-p (0.95) are fixed following Fried et al. (2023) without exploring alternatives. While all five prompt configurations are reported, the total compute budget for the prompt comparison is not stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Tables 8-10 (Appendix A.4) report all prompt configurations for every model. The paper selects the best-performing prompt per model with a clear criterion (highest Pass@1 with truncation), with all alternatives transparently shown."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many cross-model comparisons are drawn across 15+ models on multiple task categories and prompts, but no statistical tests are performed at all, so no multiple comparison correction is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the SAFIM benchmark and evaluate all models on it. They do not discuss the potential bias of evaluating their own benchmark construction, post-processing, and prompt design choices."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Figure 3 plots model size vs. performance, but actual compute budgets (inference cost, GPU hours) are not reported. Models of vastly different sizes and architectures are compared without controlling for compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 extensively discusses what existing benchmarks measure vs. real development needs and how SAFIM addresses the gap. Each task category's construct is explained (Section 3.2). Appendix A.6 shows Phi models' SAFIM results differ from HumanEval, demonstrating SAFIM captures different capabilities."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. All models are evaluated via direct inference (single-turn generation) with standardized prompts."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses code from April 2022 - January 2023, deliberately after most models' training cutoffs. Table 1 shows training cutoffs for each model. Appendix A.9 creates a temporally separated test set (April 2023 - January 2024) to further validate that temporal leakage is not affecting results."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly discuss whether the FIM prompts (prefix + suffix context) could leak answer information beyond what would be available in realistic code completion scenarios. The suffix context is part of the task design but its potential to provide unintended hints is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While within-dataset deduplication is performed (CodeBLEU > 0.9 threshold), the paper does not discuss whether test examples share structural similarities with training data beyond temporal separation (e.g., similar Codeforces problems, same algorithm patterns)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.9 uses a concrete temporal split method: creates a new test set from a later time period (April 2023 - January 2024) with zero date overlap with any model's training data. Table 17 compares original vs. new dataset performance, and Figure 5 shows performance stability across months, constituting a concrete leakage detection approach."
       }
     }
   },
   "claims": [
     {
       "claim": "FIM pretraining not only enhances FIM proficiency but also improves Left-to-Right (L2R) inference performance.",
       "evidence": "Table 2 shows FIM-pretrained StarCoder (29.3% L2R) outperforms purely L2R-pretrained CodeGen-16B (24.6% L2R) on algorithmic block completion. CodeLLaMa-34B (no FIM) underperforms smaller FIM-trained CodeLLaMa-13B (38.5% vs 41.4%).",
       "supported": "moderate"
     },
     {
       "claim": "Pretraining methods and data quality have more impact than model size on code FIM performance.",
       "evidence": "Table 4/Figure 3: StarCoder (15.5B, 55.5% avg) matches GPT-4 (>220B, 53.3% avg). DeepSeekCoder-1.3B (52.6%) outperforms GPT-3.5 (175B, 40.9%). Within CodeLLaMa family, 7.8-point size spread vs. 14-point cross-family gap (CodeLLaMa vs CodeGen).",
       "supported": "moderate"
     },
     {
       "claim": "Syntax-aware truncation significantly enhances FIM output quality and enables fair model comparison.",
       "evidence": "Table 3: CodeLLaMa-13B Pass@1 jumps from 16.4% to 41.4% with truncation; CErr% drops from 64.6% to 10.9%. CodeGen-16B goes from 0.0% to 25.9% Pass@1. All models show improvement or stability.",
       "supported": "strong"
     },
     {
       "claim": "Prompt selection is crucial for fair evaluation, as different models respond differently to different prompt types.",
       "evidence": "Table 2: CodeGen-16B achieves 25.9% with SPM but only 15.2% with IPF. CodeLLaMa-13B achieves 41.4% with SPM but only 10.2% with PSM. DeepSeekCoder-33B achieves 60.8% with PSM but 33.8% with IPF.",
       "supported": "strong"
     },
     {
       "claim": "Data contamination has negligible impact on SAFIM evaluation results despite date-range overlaps for some models.",
       "evidence": "Appendix A.9, Table 17: On a new test set (April 2023 - January 2024) with no date overlap for any model, all models maintain or improve performance (e.g., DeepSeekCoder-33B: 60.8% → 61.7%). Figure 5 shows stable performance across months.",
       "supported": "moderate"
     },
     {
       "claim": "Repository-level pretraining data enhances API function call completion, while execution-based feedback improves control-flow completion.",
       "evidence": "Table 4: StarCoder and DeepSeekCoder (with repo-level context) lead in API completion (68.1%, 75.2%). CodeLLaMa (with execution feedback) shows relatively strong control-flow performance (57.2% for 13B) compared to its API performance (59.7%).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or statistical tests",
       "detail": "All comparisons across 15+ models are made on single-run point estimates without any uncertainty quantification, confidence intervals, or statistical significance tests. With stochastic sampling (temperature 0.2, top-p 0.95), output varies across runs, but only one completion per example is generated. The paper argues dataset size (17,720) compensates, but this does not address model-level variance."
     },
     {
       "flag": "Causal claims from observational data",
       "detail": "The paper's headline claim that 'FIM pretraining enhances L2R inference' is drawn from comparing models that differ in architecture, training data volume, training data quality, and pretraining objectives simultaneously. The paper acknowledges this in Section 7 but presents the causal claims prominently in the abstract without caveats."
     },
     {
       "flag": "Potential conflict of interest",
       "detail": "Two of four authors are from Meta (AI at Meta), the paper is partially funded by a Meta gift, and Meta's CodeLLaMa is among the evaluated models. While CodeLLaMa does not consistently outperform competitors, no competing interests statement is provided."
     },
     {
       "flag": "API function call split is very small",
       "detail": "The API function call completion split contains only 310 examples (Table 5), compared to 8,781 and 8,629 for the other two splits. This small sample size for one-third of the benchmark's task categories may not provide robust estimates, especially for per-language breakdowns."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "doi": "10.48550/arXiv.2107.03374",
       "relevance": "Introduces HumanEval and Codex, foundational benchmark and model for code generation evaluation."
     },
     {
       "title": "InCoder: A generative model for code infilling and synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan", "Jessy Lin"],
       "year": 2023,
       "arxiv_id": "2204.05999",
       "doi": "10.48550/arXiv.2204.05999",
       "relevance": "Pioneered FIM as a pretraining objective for decoder-only code LLMs and established HumanEval-Infilling benchmark."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "doi": "10.48550/arXiv.2308.12950",
       "relevance": "Major open-source code LLM with mixed FIM+L2R pretraining, key evaluated model in this study."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "doi": "10.48550/arXiv.2305.06161",
       "relevance": "Large open-source code LLM with FIM pretraining and repository-level context, strong performer in benchmark evaluations."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming – the rise of code intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "doi": "10.48550/arXiv.2401.14196",
       "relevance": "Top-performing code LLM in the SAFIM benchmark, demonstrates impact of repository-level pretraining and FIM objectives."
     },
     {
       "title": "SWE-Bench: Can language models resolve real-world Github issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "doi": "10.48550/arXiv.2310.06770",
       "relevance": "Real-world software engineering benchmark for LLMs, cited as a contextually rich benchmark in the related work."
     },
     {
       "title": "Efficient training of language models to fill in the middle",
       "authors": ["Mohammad Bavarian", "Heewoo Jun", "Nikolas Tezak"],
       "year": 2022,
       "arxiv_id": "2207.14255",
       "doi": "10.48550/arXiv.2207.14255",
       "relevance": "Foundational work on FIM training methodology showing 90% FIM ratio doesn't harm L2R performance, directly relevant to this paper's findings."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "doi": "10.48550/arXiv.2108.07732",
       "relevance": "Introduces MBPP benchmark for code generation, one of the key existing benchmarks discussed in related work."
     },
     {
       "title": "Codegen: An open large language model for code with multi-turn program synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi"],
       "year": 2023,
       "arxiv_id": "2203.13474",
       "doi": "10.48550/arXiv.2203.13474",
       "relevance": "Open L2R-pretrained code LLM used as a key baseline demonstrating limitations of pure L2R pretraining for FIM tasks."
     },
     {
       "title": "RepoBench: Benchmarking repository-level code auto-completion systems",
       "authors": ["Tianyang Liu", "Canwen Xu", "Julian McAuley"],
       "year": 2023,
       "arxiv_id": "2306.03091",
       "doi": "10.48550/arXiv.2306.03091",
       "relevance": "Repository-level code completion benchmark, contextually related to evaluating LLMs' ability to use cross-file context."
     },
     {
       "title": "Rethinking benchmark and contamination for language models with rephrased samples",
       "authors": ["Shuo Yang", "Wei-Lin Chiang", "Lianmin Zheng"],
       "year": 2023,
       "arxiv_id": "2311.04850",
       "doi": "10.48550/arXiv.2311.04850",
       "relevance": "Directly addresses benchmark contamination in LLM evaluation, a key concern that motivates SAFIM's temporal cutoff design."
     },
     {
       "title": "WizardCoder: Empowering code large language models with Evol-Instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "doi": "10.48550/arXiv.2306.08568",
       "relevance": "Instruction-tuned code LLM evaluated in extended experiments, showing finetuning on distilled data can hurt FIM performance."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a publicly available benchmark toolkit and dataset for evaluating code LLMs on FIM tasks, useful for model developers but not directly usable by most practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that smaller models can match larger ones and FIM helps L2R is mildly surprising but aligns with a growing body of evidence on data quality over model size."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the Impact Statement mentions risks of automated code generation but this is generic."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy — a straightforward benchmark paper with balanced evaluation across multiple model families."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code, dataset, and leaderboard (safimbenchmark.com) are publicly available; researchers can evaluate their own models on SAFIM."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Published at ICML 2024 (top venue), evaluates well-known models (GPT-4, CodeLLaMa, DeepSeekCoder), two authors from Meta AI."
     }
   }
 }