ai-research-survey

scan.json (28201B)

---

 {
   "paper": {
     "title": "Statically Contextualizing Large Language Models with Typed Holes",
     "authors": ["Andrew Blinn", "Xiang Li", "June Hyung Kim", "Cyrus Omar"],
     "year": 2024,
     "venue": "Proc. ACM Program. Lang. (OOPSLA)",
     "arxiv_id": "2409.00921",
     "doi": "10.1145/3689728"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states that an artifact is available on Zenodo (reference [11]) containing MVUBench, raw experiment data, testing harness, Hazel source, and StarCoder2 model. Hazel source is also on GitHub at https://github.com/hazelgrove/hazel/."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states the Zenodo artifact contains 'the MVUBench program sketches and solutions, the raw data of our experiments.' The artifact is password-protected to prevent scraping."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, or detailed environment specifications. The paper mentions StarCoder2-15B can run locally but does not provide environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While an artifact is released, the paper does not describe step-by-step reproduction instructions. The Zenodo artifact description and artifact review process may contain such instructions, but the paper itself does not."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as averages across 20 trials (Figures 8, 16, 18, 19) but no confidence intervals or error bars are shown. Only point estimates are provided in the bar charts."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., types + headers outperform vector retrieval) based on comparing averages across 20 trials without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context: 'headers have a large multiplicative effect, increasing test performance threefold' (Section 2.9), 'error rounds... increasing performance by a factor of 4 for types without headers, and a factor of 1.5 for both types and headers.' Ratio of with-vs-without headers is reported as 3 for Hazel and 1.5 for TypeScript."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 20 trials per configuration but does not justify why 20 was chosen or discuss whether this is sufficient for statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 20 trials but no standard deviations, interquartile ranges, or other spread measures are reported. The paper notes model non-determinism (reference [56]) but does not quantify variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are included: no-context (Section 2.8.3), exhaustive retrieval (Section 2.8.4), and vector retrieval with confounds (Section 2.8.5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Vector retrieval using OpenAI Ada embeddings represents a standard contemporary RAG approach. Exhaustive retrieval represents the upper bound. These are reasonable baselines for 2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.8.2 describes a systematic feature ablation with 8 configurations (2×2×2) varying type retrieval, header retrieval, and error correction rounds independently."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage of tests passed (correctness), token usage (Figure 15), and time elapsed (Figure 14) as separate metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated code is included. Evaluation is entirely automated via unit tests. Given claims about code quality and developer productivity, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "MVUBench was constructed from scratch specifically to avoid data contamination (Section 1.1.2). The test suites (10-15 tests per application) are held out from the prompt context."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per application (TO, BO, EM, PL, PA) in Figures 8, 16, 18, and 19, showing per-task performance rather than only averages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Several failure cases are discussed with examples: Figure 7 (hallucinated types), Figure 9 (various failure modes), Figure 12 (confounding vector retrieval chunk), Figure 13 (exotic completions). StarCoder2 degradation with headers for BO and TO is analyzed in Section 2.10."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.10 reports that headers degraded performance for BO and TO with StarCoder2. Error rounds alone without type context are shown to be ineffective. Vector retrieval's confounding issues are documented."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that type definitions are 'particularly impactful' and that the techniques are validated in both Hazel and TypeScript — both supported by the ablation results in Figures 8, 16, 18, 19."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'type definitions improve completion quality' are supported by the ablation study design (Section 2.8.2), which systematically manipulates individual features in a controlled manner."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims these methods apply to languages generally ('language servers can implement to expose capabilities'). While the paper tests Hazel and TypeScript, MVUBench consists of only 5 small MVU applications — the generalization to arbitrary programming tasks, other languages, and larger codebases is not adequately bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 (Threats to Validity) discusses several alternative explanations: headers' effectiveness depends on functions already being implemented, MVUBench is not representative of all coding tasks, the no-context baseline may be unrealistically weak, and the RAG baseline is simplistic."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures unit test pass rates and discusses this as a measure of functional correctness of code completions, not broader claims about developer productivity. The claims match the granularity of the measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4-0613 is specified by exact checkpoint (Section 1.1.3, Section 2.9.1). StarCoder2-15B is specified by name and size. Ada embedding model is identified as text-embedding-ada-002 (Section 2.8.5)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 provides substantial prompt detail including specific instructions (quoted), few-shot examples (quoted), and negative characterization rules (quoted). The static retrieval prompt construction is fully described in Sections 2.5-2.6 with concrete examples."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 0.6 is stated (Section 2.8.2), with justification ('selected based on trial experiments as a balance between noisy variance and producing a range of interestingly distinct completions'). Context window sizes (8k for GPT-4-0613, 16k for StarCoder2) are noted."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The full 'trialogue' architecture is described in Section 2.3 with detailed steps (system message, user message with static retrieval, model response, error correction loop). Figure 1 diagrams the conversational architecture. The error correction loop is capped at 2 iterations (Section 2.7)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.8.1 describes how MVUBench was constructed. Section 2.8.5 describes how the vector retrieval database was created (150-character chunks, 6 entries retrieved, cosine similarity). The static retrieval process is documented in Sections 2.5-2.6."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 'Threats to Validity' provides a dedicated discussion of limitations spanning multiple paragraphs covering representativeness, baseline appropriateness, and RAG simplicity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 identifies specific threats: header effectiveness depends on pre-existing implementations, MVUBench is MVU-specific, the no-context baseline is unrealistically weak, the RAG baseline uses simplistic chunking, and the combined codebase for vector retrieval may not represent real large-scale codebases."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly states: 'MVUBench is not (and is not meant to) be representative of all coding tasks' and notes the TypeScript port is 'a very close translation' raising applicability questions. The paper notes the context size delta is 'not reflective of real-world use cases' (Section 2.9)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states the Zenodo artifact contains 'the raw data of our experiments.' This allows independent verification of the reported results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.8.1 describes MVUBench construction: 5 MVU applications written from scratch, each with 10-15 tests, designed to avoid data contamination. Section 2.8.2 describes the 320-trial experiment design."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The benchmark is constructed by the authors and evaluated automatically."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: program sketch → static retrieval from language server → prompt construction → LLM generation → error correction loop → test evaluation. Each step is described in Sections 2.3-2.7."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 states: 'This work was partially funded by the National Science Foundation (Award #2238744).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from the University of Michigan and are the developers of Hazel. Their affiliation with the evaluated system is clear from the paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is an independent government funding agency with no financial stake in Hazel's success or in the experimental outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included in the paper. The authors evaluate their own system (Hazel) but do not declare whether they hold any financial interests related to it."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state GPT-4-0613's training cutoff date. For StarCoder2, they mention The Stack v2 but do not state a specific cutoff. This is relevant because they claim MVUBench avoids contamination."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1.2 extensively discusses data contamination risks with existing benchmarks and explains that MVUBench was 'conceptualized and implemented from scratch, without directly adapting any code from GitHub.' Section 9 acknowledges even their benchmark may have been exposed via Copilot-enabled IDEs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination by constructing MVUBench from scratch (Section 1.1.2), password-protecting the Zenodo artifact (Section 8), and planning to 'control the release of these benchmarks to limit the likelihood of future contamination' (Section 1.1.2)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 14 reports time elapsed per trial. Figure 15 reports total tokens used (both sent and received). Section 2.9.1 notes 'precisely proportional to the total cost' and discusses practical latency concerns."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total API spend or compute budget for all 320+ trials is not stated. Token counts are shown per configuration but total expenditure across the full experiment is not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "20 trials per configuration are run to account for 'model non-determinism' (Section 2.8.2) but no analysis of seed sensitivity or variance across seeds is reported. Only averages are shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.8.2 clearly states '20 completion trials per combination (to account for model non-determinism)' and that the experiment consists of 320 total trials."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.6 was 'selected based on trial experiments' (Section 2.8.2) but the number of configurations tried and selection method are not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper does not cherry-pick a best configuration — it reports all 8 ablation configurations plus 3 baselines, showing the full landscape. Temperature selection is briefly justified."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is moot."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (Hazel Assistant with their language server) but do not acknowledge the bias of author-evaluation. They also constructed MVUBench themselves."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figures 14 and 15 show time and token cost per configuration, allowing comparison of performance vs. compute. Section 2.9 notes the context size delta between static retrieval (890 chars avg) and exhaustive retrieval (1370 chars avg)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1.2 provides detailed discussion of why existing benchmarks (HumanEval, MBPP, RepoEval, etc.) are unsuitable and why MVU applications test the specific capability claimed (semantic contextualization). Section 4 notes MVUBench is not representative of all tasks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates its own integrated system (Hazel Assistant) — the scaffold IS the thing being tested, not a confound between model comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "MVUBench was constructed from scratch specifically to address this: 'conceptualizing and implementing these applications from scratch, without directly adapting any code from GitHub' (Section 1.1.2). The benchmark did not exist before the models' training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (e.g., providing type information) constitutes feature leakage relative to real-world usage scenarios where such information might not be as cleanly available."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 5 MVU applications in MVUBench share structural patterns (all MVU architecture). No discussion of whether this structural similarity between benchmark items affects the evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The primary leakage prevention method is constructing MVUBench from scratch and password-protecting the artifact (Section 8). Section 9 acknowledges even this may not be fully clean due to Copilot-enabled IDEs during development."
       }
     }
   },
   "claims": [
     {
       "claim": "Type definitions are particularly impactful for LLM code completion contextualization.",
       "evidence": "Ablation study (Section 2.9, Figure 8): adding type definitions dramatically improves test pass rates from near-zero to substantial completion quality across all 5 MVU applications for both GPT-4 and StarCoder2.",
       "supported": "strong"
     },
     {
       "claim": "Function headers have a large multiplicative effect when combined with type definitions, increasing test performance threefold in Hazel.",
       "evidence": "Section 2.9, Figure 8: types+headers outperforms types-only by approximately 3x for Hazel GPT-4. This ratio is 1.5 for TypeScript (Section 3.2).",
       "supported": "moderate"
     },
     {
       "claim": "Error correction rounds are ineffective alone but multiply effectiveness when combined with static retrieval.",
       "evidence": "Section 2.9: error rounds alone are ineffective but multiply performance by 4x (types only) and 1.5x (types+headers) in Hazel. Less impactful in TypeScript (1.2x ratio, Section 3.2).",
       "supported": "moderate"
     },
     {
       "claim": "Static retrieval outperforms vector retrieval for semantic contextualization.",
       "evidence": "Figures 8, 18: types+headers outperform vector retrieval baseline. However, the paper acknowledges a confounding chunk (Figure 12) disproportionately affected vector retrieval results (Section 2.9).",
       "supported": "weak"
     },
     {
       "claim": "These techniques transfer from low-resource (Hazel) to high-resource (TypeScript) languages.",
       "evidence": "Section 3, Figures 18-19: TypeScript results show similar trends though with smaller margins, as the model is already more capable in TypeScript.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Static type-based contextualization significantly improves LLM code completion quality on high-context tasks. Type definitions are the most critical context to provide, with function headers and error correction loops providing additional multiplicative benefits. These results hold for both a large closed model (GPT-4) and a smaller open model (StarCoder2-15B), and transfer from a low-resource language (Hazel) to TypeScript. The paper proposes ChatLSP as a standard interface for language servers to expose this kind of static context to AI assistants.",
   "red_flags": [
     {
       "flag": "Authors evaluate their own system",
       "detail": "All four authors are Hazel developers at the University of Michigan. They designed MVUBench, the Hazel Language Server, the Hazel Assistant, and conducted all evaluations. No independent evaluation or third-party replication is included."
     },
     {
       "flag": "Tiny benchmark scale",
       "detail": "MVUBench consists of only 5 small MVU applications with 10-15 tests each. The generalization to real-world codebases of hundreds of thousands of lines, non-MVU tasks, and other programming paradigms is unclear."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Despite running 20 trials per configuration, no confidence intervals, error bars, or significance tests are reported. Claims about relative performance are based on comparing averages without quantifying uncertainty."
     },
     {
       "flag": "Simplistic RAG baseline may be unfairly weak",
       "detail": "The vector retrieval baseline uses naive 150-character uniform chunking. The authors acknowledge more sophisticated chunking strategies exist but chose the simplest approach. Section 4 concedes this is arguable."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for LLM code generation evaluation."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for LLM program synthesis."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "authors": ["Fengji Zhang", "Bei Chen"],
       "year": 2023,
       "doi": "10.18653/v1/2023.emnlp-main.151",
       "relevance": "Vector retrieval approach for repository-level code completion — direct comparison point."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark addressing data leakage concerns."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Open-source code LLM used as one of the two evaluation models."
     },
     {
       "title": "CoCoMIC: Code Completion by Jointly Modeling In-file and Cross-file Context",
       "authors": ["Yangruibo Ding", "Zijian Wang"],
       "year": 2024,
       "relevance": "Framework for joint in-file and cross-file context for code completion."
     },
     {
       "title": "Repair is nearly generation: multilingual program repair with LLMs",
       "authors": ["Harshit Joshi", "José Cambronero Sanchez"],
       "year": 2023,
       "doi": "10.1609/aaai.v37i4.25642",
       "relevance": "LLM-based program repair using iterative error correction loops."
     },
     {
       "title": "Enhancing LLM-Based Coding Tools through Native Integration of IDE-Derived Static Context",
       "authors": ["Yichen Li", "Yun Peng"],
       "year": 2024,
       "arxiv_id": "2402.03630",
       "relevance": "Closely related work using IDE/language server static context for LLM code completion."
     },
     {
       "title": "STALL+: Boosting LLM-based Repository-level Code Completion with Static Analysis",
       "authors": ["Junwei Liu", "Yixuan Chen"],
       "year": 2024,
       "arxiv_id": "2406.10018",
       "relevance": "General framework for applying static analysis to repository-level code completion."
     },
     {
       "title": "De-Hallucinator: Iterative Grounding for LLM-Based Code Completion",
       "authors": ["Aryaz Eghbali", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2401.01701",
       "relevance": "Post-generation semantic lookup to fix hallucinated code definitions."
     },
     {
       "title": "Copiloting the Copilots: Fusing Large Language Models with Completion Engines for Automated Program Repair",
       "authors": ["Yuxiang Wei", "Chunqiu Steven Xia"],
       "year": 2023,
       "doi": "10.1145/3611643.3616271",
       "relevance": "Combines LLMs with semantic code completion for program repair."
     },
     {
       "title": "RLCoder: Reinforcement Learning for Repository-Level Code Completion",
       "authors": ["Yanlin Wang", "Yanli Wang"],
       "year": 2024,
       "arxiv_id": "2407.19487",
       "relevance": "RL-based ranking of retrieved code snippets for repository-level completion."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Key study on AI coding assistant productivity impact."
     }
   ]
 }