ai-research-survey

scan-v4.json (23798B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks",
     "authors": [
       "Linyuan Gong",
       "Sida Wang",
       "Mostafa Elhoushi",
       "Alvin Cheung"
     ],
     "year": 2024,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2403.04814",
     "doi": "10.48550/arXiv.2403.04814"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: FIM pretraining enhancing FIM and L2R inference is shown in Table 2 (Section 6.1), pretraining methods mattering more than model size is demonstrated in Table 4/Figure 3 (Section 6.3), SAFIM providing fair comparisons is evidenced by the truncation ablation (Table 3) and prompt comparison (Table 2).",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'FIM pretraining not only enhances FIM proficiency but also improves Left-to-Right (L2R) inference' — a causal claim based on observational comparison across model families with different architectures, data, and training procedures. While the paper acknowledges in Section 7 that 'our conclusions are drawn from comparisons across various model families trained with different paradigms, rather than from controlled experiments,' the abstract presents these causal claims without caveats.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title scopes to 'Syntax-Aware Code Fill-in-the-Middle Tasks.' Claims use hedging language ('our findings suggest'). The paper acknowledges cross-family comparisons are not controlled experiments (Section 7) and that results may vary by programming language (Appendix A.7). Scope is mostly bounded to SAFIM's three task types and four languages.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple alternative explanations: data contamination (Appendix A.9 with a dedicated experiment), differences in pretraining environments confounding cross-family comparisons (Section 1, Section 7), programming language coding style affecting results (Appendix A.7), and training data cutoff date overlap (Table 1).",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@1 on execution-based tests and syntactical matching, and frames results in terms of 'code completion proficiency' and 'FIM performance.' The measurements match the granularity of claims — no broader framing of 'developer productivity' or similar proxy gaps exists.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 contains substantive limitations discussion: 'We acknowledge a key limitation in our study: our conclusions are drawn from comparisons across various model families trained with different paradigms, rather than from controlled experiments altering pretraining paradigms within the same model.' An Impact Statement section also discusses broader concerns.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: data contamination from training data overlap (Table 1, Appendix A.9 with a dedicated experiment), non-controlled cross-family comparisons (Section 7), programming language coding style affecting results (Appendix A.7), and prompt sensitivity affecting model rankings (Section 6.1).",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what the results do NOT show: 'our conclusions are drawn from comparisons across various model families trained with different paradigms, rather than from controlled experiments' and proposes 'future work in pretraining such models under the same environment to validate these observations further.' The paper also scopes to decoder-only models, excluding encoder-decoder models (Section 2).",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section lists: 'gift from Meta, the U.S. National Science Foundation through grants IIS-1955488, IIS-2027575, ARO W911NF2110339, ONR N00014-21-1-2724, and DOE awards DE-SC0016260, DE-SC0021982.'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Linyuan Gong and Alvin Cheung at University of California at Berkeley, Sida Wang and Mostafa Elhoushi at 'AI at Meta.' Two authors are at Meta, whose CodeLLaMa model is evaluated.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta provided a gift fund and two authors are Meta employees. Meta develops CodeLLaMa, one of the evaluated models. While CodeLLaMa does not consistently win (DeepSeekCoder outperforms it), the funder has a financial interest in LLM evaluation outcomes.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Two Meta-affiliated authors evaluate Meta's CodeLLaMa, but no explicit conflict-of-interest disclosure is provided.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are well-defined: FIM (Fill-in-the-Middle) is explained with all five prompt variants (L2R, PSM, SPM, IPF, 1S), 'syntax-aware' is defined as AST-based masking, and each of the three benchmark splits (algorithmic block, control-flow expression, API function call) is precisely characterized with examples.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states three contributions: (1) the SAFIM benchmark dataset with 17,720 examples, (2) multiple prompt designs enabling fair cross-model comparisons, and (3) a novel syntax-aware truncation algorithm for post-processing.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides substantive engagement with prior FIM benchmarks (HumanEval-Infilling, HumanEval, MBPP), code LLMs, and repository-level benchmarks, explicitly identifying gaps SAFIM addresses: scale, multilingual coverage, execution-based evaluation, and syntax-awareness.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that AST-based syntax-aware masking better represents real development tasks than randomly masked lines or character spans, contrasting explicitly with HumanEval-Infilling's approach. Each split is motivated by a distinct code comprehension capability (algorithm design, control-flow understanding, API knowledge).",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "The paper provides statistics on example sizes and average completion lengths (Tables 5–6, Figure 4) but does not characterize difficulty distribution into tiers (easy/medium/hard) or empirically measure per-example difficulty.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "No explicit ceiling or floor effect analysis is conducted. The observed Pass@1 range of ~23%–69% suggests reasonable discrimination, but this is observed post-hoc rather than verified by design.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human performance baseline is reported. The API split mentions examples are 'solvable by humans based on the given context' but no human performance measurement is provided for any split.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Execution-based Pass@1 is justified for algorithmic/control-flow tasks (unit tests available), syntactic match for API calls is justified by the impossibility of executing external API side effects, and the use of single Pass@1 rather than multi-sample averaging is justified by the dataset's large scale (17,720 examples).",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "SAFIM uses a post-April 2022 cutoff to avoid overlap with The Stack (cutoff March 2022) and GPT-3.5/4 training data (cutoff September 2021). Appendix A.9 empirically validates that even for models with overlapping cutoffs (CodeLLaMa, DeepSeekCoder), performance is essentially unchanged on a clean post-April 2023 held-out dataset.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether or how the benchmark will remain discriminative as model capabilities advance, or provide a plan for updating the benchmark when model training cutoffs surpass the data sources.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper briefly notes that syntactic match evaluation for API calls is imperfect and that cross-family comparisons aren't controlled, but does not systematically enumerate benchmark failure modes or describe what behaviors SAFIM cannot measure or what strategies could game it.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The full evaluation toolkit and dataset are released at github.com/gonglinyuan/safim; Appendix A.3 (Table 7) provides exact model identifiers for every evaluated model on Huggingface and OpenAI API, and generation hyperparameters (top-p=0.95, temperature=0.2) are specified.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Dataset construction is documented with source description (Codeforces and GitHub), date ranges (April 2022–January 2023), filtering criteria (unit test pass rate, CodeBLEU deduplication, GitHub quality filters), and per-split and per-language statistics in Tables 5–6 and Figure 4.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The paper states the dataset is available at GitHub and lists a leaderboard URL but specifies no license for the benchmark data or evaluation toolkit, creating uncertainty about legal reuse and redistribution.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Intended use is clearly stated: evaluating code LLMs on FIM tasks and comparing pretraining paradigms. The paper also explicitly cautions that cross-family comparisons 'should be interpreted with caution' and that controlled experiments are needed to validate causal conclusions.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "FIM pretraining enhances both FIM task performance and Left-to-Right (L2R) inference capabilities.",
       "evidence": "Table 2 shows FIM-pretrained StarCoder (15.5B) outperforms L2R-only CodeGen-16B in L2R mode (29.3% vs 24.6%); CodeLLaMa-13B (FIM+L2R) outperforms CodeLLaMa-34B (L2R only) on FIM tasks despite smaller size.",
       "supported": "moderate"
     },
     {
       "claim": "Pretraining methods and data quality matter more than sheer model size for code task performance.",
       "evidence": "StarCoder (15.5B) matches GPT-4 (53.3% vs 55.5% avg Pass@1); DeepSeekCoder-1.3B (52.6%) far exceeds CodeGen-16B (31.0%). Within-family scaling gains are modest; cross-family differences are large.",
       "supported": "moderate"
     },
     {
       "claim": "Syntax-aware truncation significantly improves Pass@1 and reduces compilation errors versus no truncation.",
       "evidence": "Table 3: CodeLLaMa-13B jumps from 16.4% to 41.4% Pass@1 with CErr% dropping from 64.6% to 10.9%; CodeGen-16B goes from 0.0% to 25.9%. Improvements are consistent across all models.",
       "supported": "strong"
     },
     {
       "claim": "Using a limited prompt range skews FIM benchmark comparisons; multiple prompt types are required for fair evaluation.",
       "evidence": "Table 2 shows CodeGen-16B scores 15.2% on IPF but 25.9% on SPM, reversing its apparent ranking versus InCoder-6B (25.2% on PSM) — exactly replicating the skewed comparison methodology of Fried et al. 2023.",
       "supported": "strong"
     },
     {
       "claim": "Data contamination (date-range overlap) has negligible impact on SAFIM evaluation results.",
       "evidence": "Appendix A.9 Table 17 shows no significant performance decrease on a clean post-April 2023 dataset for CodeLLaMa or DeepSeekCoder, with several models slightly improving on newer data.",
       "supported": "moderate"
     },
     {
       "claim": "Repository-level pretraining context is the key factor for API function call completion performance.",
       "evidence": "StarCoder (68.1%) and DeepSeekCoder (75.2%) lead the API split; the paper attributes this to their repo-level pretraining (GitHub issues/commits and topological dependency ordering respectively). But this is inferred from observational model comparisons without ablation.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "SAFIM introduces a 17,720-example multilingual code FIM benchmark using AST-based masking across three task types (algorithmic block, control-flow, API function call). Evaluation of 15+ LLMs shows DeepSeekCoder-33B achieves the highest performance (69.0% avg Pass@1) while the novel syntax-aware truncation algorithm dramatically improves scores for models lacking explicit stop tokens (e.g., CodeLLaMa-13B: 16.4%→41.4%). Observationally, FIM-pretrained smaller models match or outperform larger L2R-only models, and prompt selection has a large impact on apparent model rankings — prior benchmarks using a single prompt type produced misleading comparisons. The contamination analysis found negligible impact even for models with overlapping training date ranges.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "No human performance is measured on any SAFIM split, making it impossible to calibrate benchmark difficulty or know whether top models approach human-level performance."
     },
     {
       "flag": "Meta conflict of interest unacknowledged",
       "detail": "Two authors are employed at 'AI at Meta' and the work is funded by a Meta gift, yet the paper evaluates CodeLLaMa (Meta's model) without any competing interests declaration or mitigation."
     },
     {
       "flag": "Causal claims from uncontrolled comparisons",
       "detail": "The abstract frames 'FIM pretraining improves L2R inference' and 'pretraining > model size' as findings, but these are observational cross-family comparisons confounded by training data, compute, and architecture differences — the paper acknowledges this only in the conclusion body."
     },
     {
       "flag": "API split very small (310 examples)",
       "detail": "The API function call completion split has only 310 examples from GitHub versus 8,781 and 8,629 in the other two splits. Conclusions about API performance are based on a much weaker statistical foundation."
     },
     {
       "flag": "No licensing information",
       "detail": "The paper provides no license for the benchmark data or toolkit, creating legal ambiguity about use and redistribution for others who wish to build on SAFIM."
     },
     {
       "flag": "No difficulty distribution or ceiling/floor analysis",
       "detail": "Benchmark items are not characterized by difficulty tier, and no explicit ceiling/floor effect check is performed — though the observed score range (23%–69%) suggests adequate discrimination in practice."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval / Codex)",
       "relevance": "The primary prior benchmark SAFIM extends and critiques for limited scope (164 Python functions, standalone generation only, no FIM)"
     },
     {
       "title": "Efficient Training of Language Models to Fill in the Middle (Bavarian et al. 2022)",
       "relevance": "Establishes FIM pretraining methodology and the HumanEval-Infilling benchmark that SAFIM supersedes"
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "relevance": "Key evaluated model; also the source of skewed prior evaluation methodology that SAFIM's multi-prompt approach corrects"
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "relevance": "Top-performing evaluated model; demonstrates benefits of repo-level pretraining for code tasks"
     },
     {
       "title": "StarCoder: May the Source Be with You!",
       "relevance": "Evaluated model incorporating GitHub issues/commits in pretraining; demonstrates FIM pretraining benefits for both FIM and L2R"
     },
     {
       "title": "InCoder: A Generative Model for Code Infilling and Synthesis",
       "relevance": "Pioneer of FIM pretraining for decoder-only models; SAFIM directly extends and critiques the HumanEval-Infilling benchmark introduced here"
     },
     {
       "title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
       "relevance": "Motivates SAFIM's contamination-resistance design via temporal cutoffs"
     },
     {
       "title": "xCodeEval: A Large Scale Multilingual Multitask Benchmark for Code Understanding, Generation, Translation and Retrieval",
       "relevance": "Provides the ExecEval execution framework used for SAFIM's execution-based evaluation"
     },
     {
       "title": "RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems",
       "relevance": "Related repository-level code completion benchmark; contextualizes SAFIM's API function call split"
     },
     {
       "title": "SWE-Bench: Can Language Models Resolve Real-World Github Issues?",
       "relevance": "Contextualizes SAFIM in the broader landscape of real-world software engineering benchmarks"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a publicly available benchmark toolkit and dataset for evaluating code LLMs on FIM tasks, useful for model developers but not directly usable by most practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that smaller models can match larger ones and FIM helps L2R is mildly surprising but aligns with a growing body of evidence on data quality over model size."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the Impact Statement mentions risks of automated code generation but this is generic."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy — a straightforward benchmark paper with balanced evaluation across multiple model families."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code, dataset, and leaderboard (safimbenchmark.com) are publicly available; researchers can evaluate their own models on SAFIM."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Published at ICML 2024 (top venue), evaluates well-known models (GPT-4, CodeLLaMa, DeepSeekCoder), two authors from Meta AI."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "40881654",
         "title": "LLM Agents can Autonomously Exploit One-day Vulnerabili-ties [pdf]",
         "points": 4,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40881654"
       },
       {
         "hn_id": "40138889",
         "title": "LLM Agents Can Autonomously Exploit One-Day Vulnerabilities",
         "points": 4,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40138889"
       },
       {
         "hn_id": "40633364",
         "title": "LLM Agents Can Autonomously Exploit One-Day Vulnerabilities",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40633364"
       },
       {
         "hn_id": "41128425",
         "title": "Things Come from Having Many Good Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41128425"
       },
       {
         "hn_id": "40756286",
         "title": "Solving Maxwell's Equations with Non-Trainable Graph Neural Network",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40756286"
       },
       {
         "hn_id": "40679472",
         "title": "Discovering Optimization Algorithms With And For Large Language Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40679472"
       },
       {
         "hn_id": "40666270",
         "title": "Discovering Preference Optimization Algorithms with Large Language Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40666270"
       },
       {
         "hn_id": "40085930",
         "title": "LLM Agents Can Autonomously Exploit One-Day Vulnerabilities with 87% Success",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40085930"
       },
       {
         "hn_id": "39765229",
         "title": "Quantifying Contamination in Code Generation Capabilities of Language Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39765229"
       },
       {
         "hn_id": "39737870",
         "title": "LSTM-Based Machine Learning for Enhancing Storm Surge Forecasting Accuracy",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39737870"
       }
     ],
     "top_points": 4,
     "total_points": 23,
     "total_comments": 3
   }
 }