ai-research-survey

scan.json (33728B)

---

 {
   "paper": {
     "title": "Granite Code Models: A Family of Open Foundation Models for Code Intelligence",
     "authors": [
       "Mayank Mishra",
       "Matt Stallone",
       "Gaoyuan Zhang",
       "Yikang Shen",
       "Aditya Prasad",
       "Adriana Meza Soria",
       "Michele Merler",
       "Parameswaran Selvam",
       "Saptha Surendran",
       "Shivdeep Singh",
       "Manish Sethi",
       "Xuan-Hong Dang",
       "Pengyuan Li",
       "Kun-Lung Wu",
       "Syed Zawad",
       "Andrew Coleman",
       "Matthew White",
       "Mark Lewis",
       "Raju Pavuluri",
       "Yan Koyfman",
       "Boris Lublinsky",
       "Maximilien de Bayser",
       "Ibrahim Abdelaziz",
       "Kinjal Basu",
       "Mayank Agarwal",
       "Yi Zhou",
       "Chris Johnson",
       "Aanchal Goyal",
       "Hima Patel",
       "Yousaf Shah",
       "Petros Zerfos",
       "Heiko Ludwig",
       "Asim Munawar",
       "Maxwell Crouse",
       "Pavan Kapanipathi",
       "Shweta Salaria",
       "Bob Calio",
       "Sophia Wen",
       "Seetharami Seelam",
       "Brian Belgodere",
       "Carlos Fonseca",
       "Amith Singhee",
       "Nirmit Desai",
       "David D. Cox",
       "Ruchir Puri",
       "Rameswar Panda"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2405.04324",
     "doi": "10.48550/arXiv.2405.04324"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Granite Code models (3B-34B parameters) trained on 3.5-4.5T tokens across 116 programming languages achieve competitive or state-of-the-art performance among open-source code LLMs across diverse tasks including code generation, fixing, explanation, editing, and translation. Granite-8B-Code-Base outperforms CodeGemma-8B by ~12 points on HumanEvalPack average despite training on fewer tokens (4.5T vs 7.5T). The models show particular strength in code explanation and fixing tasks where specialized code models like StarCoder2 and CodeGemma fall behind. All models are released under Apache 2.0 license.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link (https://github.com/ibm-granite/granite-code-models) in the abstract and states 'We release all our Granite Code models under an Apache 2.0 license for both research and commercial use.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation benchmarks used are publicly available (HumanEval, MBPP, MultiPL-E, DS1000, RepoBench, etc.). Training data is sourced from publicly available datasets (Github Code Clean, StarCoderdata) though the specific curated training set is not released as a standalone artifact."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes hardware (A100/H100 clusters) and frameworks (Megatron-LM, FlashAttention 2, NVIDIA Apex) in Sections 4.3-4.4, but does not provide versioned dependency lists, requirements.txt, Dockerfile, or sufficient detail to exactly recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While a GitHub repository is linked, the paper itself does not include commands or scripts to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All benchmark results in Tables 3-16 are point estimates (e.g., '43.9% pass@1') with no confidence intervals or error bars, despite sampling-based evaluations (50 samples for MultiPL-E, 40 for DS1000, etc.)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous 'outperforms' claims (e.g., 'outperforms the most competitive CodeGemma-8B model by almost 12 points') based solely on comparing point estimates. No statistical significance tests (p-values, t-tests, etc.) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports absolute differences with baseline context: 'outperforms CodeGemma-8B by almost 12 points on HumanEvalPack (33.2% vs 21.3%)', 'outperforms Llama-3-8B-Base by ~12 points on GSM8K and ~6 points on MATH', providing both values for context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 50 samples per prompt (MultiPL-E), 40 samples (DS1000), 10 samples (CRUXEval), or 20 completions (CanItEdit) were chosen. These numbers follow prior work conventions but are not explicitly justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite using multiple samples per evaluation (50 for MultiPL-E, 40 for DS1000, 10 for CRUXEval), only mean pass@1 values are reported. No standard deviations, interquartile ranges, or other spread measures are provided across runs or samples."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baseline comparisons with StarCoder, StarCoder2, CodeLlama, CodeGemma, Mistral, Llama-3, Gemma, Mixtral, OctoCoder, and StableCode across all benchmarks (Tables 3-16)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include models released in 2024 (Llama-3, CodeGemma, StarCoder2) which were state-of-the-art at time of writing. The paper evaluates against the most recent open-source code LLMs."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper evaluates models at different sizes (3B, 8B, 20B, 34B) but does not ablate training components (e.g., phase 2 training impact, FIM objective contribution, data filtering choices). The multi-size evaluation shows scaling but not component contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass@1, exact match, edit similarity, ExcessCode, RP@1 (robustness), Code ES, ID F1, across diverse benchmarks covering generation, fixing, explanation, editing, translation, execution, math reasoning, and function calling."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluations are fully automated using pass@1 on test suites, exact match, edit similarity, and other automated metrics. No human evaluation of code quality, usefulness, or readability is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All evaluations use established public benchmarks (HumanEval, MBPP, MultiPL-E, DS1000, RepoBench, CrossCodeEval, etc.) which serve as held-out test sets separate from any development decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-language breakdowns across tables: 6 languages for HumanEvalPack (Table 3), 18 languages for MultiPL-E (Table 4), 7 libraries for DS1000 (Table 7), per-language results for RepoBench (Table 6), CrossCodeEval (Table 8), FIM (Table 9), CRUXEval (Table 14), and per-task results for BFCL (Figures 4-5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes where Granite falls behind competitors (e.g., 'lags behind CodeGemma-7B on all categories' for ReCode, Sec 6.7) but provides no systematic error analysis or qualitative examination of failure modes. There is no discussion of what types of problems Granite systematically fails on or why."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: 'no performance improvement in scaling the model sizes from 8B to 34B' for FIM (Sec 6.1.6), Granite-8B 'lags behind CodeGemma-7B on all categories' for ReCode (Sec 6.7), Granite-3B 'falls short of StarCoder2-3B' on MBPP (Sec 6.1.3), and mixed results on CrossCodeEval and RepoBench where no single model dominates."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims Granite Code models 'consistently reaches state-of-the-art performance among available open-source code LLMs.' However, results show mixed performance: on MBPP, StarCoder2-3B and CodeGemma-7B beat their Granite counterparts; on MultiPL-E, 'there is no single model that works best at every language'; on ReCode, CodeGemma-7B beats Granite-8B. 'Consistently state-of-the-art' overstates the actual results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims without adequate justification: 'We attribute this performance to our data mixture and base model training decisions' (Sec 6.2) is a causal claim about what caused good performance, but no ablation study isolates these factors. No controlled experiments demonstrate that the data mixture or training decisions specifically caused the improvements."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Code Intelligence' broadly and the abstract claims the models are 'optimized for enterprise software development workflows,' but no enterprise environment testing is performed. The paper tests on academic benchmarks only. Claims about enterprise suitability and general code intelligence extend well beyond the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed results. For instance, performance differences could be due to training data overlap with benchmarks, differences in tokenization, or data scale effects, but none of these alternatives are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass@1 on synthetic benchmarks and frames results as evidence of 'enterprise software development' capability and 'code intelligence.' The gap between benchmark performance and real-world enterprise development utility is never acknowledged or discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model names with versions are specified for all baselines: 'Mistral-7B-v0.2', 'Mixtral-8x7B-v0.1', 'Mixtral-8x22B-v0.1', 'Llama-3-8B', etc. Own models specify exact sizes and architecture configurations in Table 1."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we adhere to the formats provided in their official examples' and use 'completion format for the base models, and instruction template for the instruction-tuned models' (Sec 6.1.1) but never provides the actual prompt text used in any experiment. The specific prompts are described only in natural language."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Extensive hyperparameters reported: optimizer (AdamW, β1=0.9, β2=0.95, weight decay=0.1), learning rates (3×10⁻⁴ to 3×10⁻⁵, cosine and exponential schedules), FIM α=0.5, evaluation settings (temperature 0.2, top-p 0.95 for MultiPL-E/DS1000; temperature 0.8 for pass@5), batch sizes (Table 1), context lengths."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly on benchmarks with standard prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents the data pipeline in detail: language filtering (116 of 300+ languages), four quality filtering rules (alphabetic threshold, XML filter, HTML visible text filter, JSON/YAML character count), exact dedup (SHA256), fuzzy dedup (MinHash + LSH, Jaccard threshold 0.7), HAP/PII filtering (StarPII model), and malware scanning (ClamAV)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion (Section 7) briefly mentions future plans (long-context variants, specialized models) but does not discuss any limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no consideration of potential issues with the evaluation methodology, data quality, or generalizability of results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show, what settings were not tested, or what claims the authors are NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The curated training dataset is not released. While source datasets (Github Code Clean, StarCoderdata) are publicly available, the specific filtered, deduplicated, and processed training set used is not available for verification. Model weights are released but not the training data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the data collection in detail: sources (publicly available GitHub datasets), filtering criteria (language filtering, quality rules), deduplication methods (SHA256 + MinHash/LSH), and content filtering (HAP/PII/malware). Natural language data sources are also described in Section 2.4."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public code repositories and established benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents the full pipeline: crawling → language filtering (116 languages from 300+) → quality filtering (4 rules) → exact dedup (SHA256) → fuzzy dedup (MinHash, Jaccard 0.7) → HAP filtering → PII redaction → malware scanning. Section 4.1 documents the two-phase training data splits."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "While there is no explicit 'Funding' section, all authors are identified as IBM Research employees, the acknowledgments thank IBM leadership by name, and the computing infrastructure (IBM's Vela and Blue Vela clusters) is described. The corporate funding source is effectively disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed under 'IBM Research' affiliation at the top of the paper. The paper is explicitly about IBM's own Granite Code models product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "IBM funded this research and has a direct commercial interest in Granite Code models performing well. The models are part of IBM's WatsonX product line (acknowledged in the paper). The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included. Given that this is a corporate product paper from IBM, the absence of a formal declaration of financial interests (patents, equity, commercial product tie-ins) is a gap."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated. The paper describes data sources but never specifies when the training data was collected or its temporal boundaries."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between the GitHub-sourced training data and the evaluation benchmarks (HumanEval, MBPP, etc.), which are also derived from publicly available code. This is a significant omission given that the training data includes public GitHub repositories."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021), MBPP (2021), and other benchmarks were published years before this work. The training data includes public GitHub code where solutions to these benchmarks are widely available. No decontamination analysis or discussion of this risk is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluations are automated benchmark tests."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported for any of the four model sizes. For an enterprise-focused model, this is a notable omission."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states carbon emissions of '~455 tCO2eq' for pretraining (Sec 4.4), describes hardware (IBM Vela A100 and Blue Vela H100 clusters), and notes instruction tuning takes 'a few hours on 8×A100 GPUs' for the 20B model. Total token counts are also provided (3.5T-4.5T)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "While multiple samples are drawn per evaluation (50 for MultiPL-E, 40 for DS1000), no variance or sensitivity across random seeds is reported. Results are single-point estimates."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Number of samples/runs is stated for most benchmarks: '50 completions per prompt' (MultiPL-E), '40 samples per each library' (DS1000), '10 samples' (CRUXEval), '20 completions per problem' (CanItEdit), '5 seeds' (ReCode). Greedy decoding benchmarks implicitly use 1 run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No mention of hyperparameter search for any aspect of training or evaluation. The paper presents a single set of training hyperparameters and evaluation settings without discussing how they were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents specific architecture choices (MHA vs GQA vs MQA, RMSNorm vs LayerNorm, swiglu vs gelu) and training configurations without justifying why these were selected over alternatives. The FIM weight α=0.5 is described as 'emperically set' but no search process is described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims across dozens of model-benchmark combinations without any correction for multiple comparisons. No statistical tests are performed at all, let alone with multiplicity corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "IBM evaluates its own Granite models against baselines re-run by the same team. While they note 'we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Sec 6), they do not acknowledge the inherent bias of authors evaluating their own system (Lucic et al., 2018 showed systematic underperformance of author re-implementations)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly compares performance relative to training tokens: 'Granite-8B-Code-Base outperforms the most competitive CodeGemma-8B model by almost 12 points on HumanEvalPack, despite being trained on significantly less number of tokens (4.5T vs 7.5T tokens).' Performance at different model sizes (3B-34B) also provides compute-performance perspective."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses many benchmarks but never discusses whether they actually measure the claimed capabilities. For example, HumanEval pass@1 is used as evidence of 'code intelligence' and 'enterprise software development' capability without questioning whether these synthetic benchmarks measure real-world coding ability."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly on benchmarks via standard prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021, MBPP in 2021, and many other benchmarks predate the training data collection. The training data includes GitHub code where solutions to these benchmarks are freely available. No temporal leakage analysis is discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information, or whether the model's training on GitHub issues and code comments provides unfair context for benchmark problems derived from similar sources."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training data comes from GitHub repositories, and several benchmarks (HumanEval, MBPP) are derived from or available on GitHub. The potential for train-test overlap from shared data sources is never discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No decontamination pipeline, canary strings, membership inference tests, or any other leakage detection method is described or applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Granite-8B-Code-Base outperforms CodeGemma-8B by almost 12 points on HumanEvalPack average (33.2% vs 21.3%), despite being trained on significantly fewer tokens (4.5T vs 7.5T).",
       "evidence": "Table 3 and Figure 1 show the HumanEvalSynthesize results. The average across 6 languages confirms the 12-point gap. Token counts are stated in Section 4.1.",
       "supported": "strong"
     },
     {
       "claim": "Granite Code models consistently reach state-of-the-art performance among available open-source code LLMs.",
       "evidence": "Tables 3-16 show competitive performance across many benchmarks, but Granite is not always best: StarCoder2-3B beats Granite-3B on MBPP (Table 5), CodeGemma-7B beats Granite-8B on ReCode (Table 16) and CRUXEval (Table 14), and 'there is no single model that works best at every language' on MultiPL-E.",
       "supported": "moderate"
     },
     {
       "claim": "Granite-8B-Code-Base outperforms Llama-3-8B-Base by ~12 points on GSM8K and ~6 points on MATH.",
       "evidence": "Table 15 shows GSM8K: 61.9 vs 49.8 (~12 point gap) and MATH: 21.4 vs 15.6 (~6 point gap).",
       "supported": "strong"
     },
     {
       "claim": "Granite Code base models significantly outperform other SOTA base code LLMs on code explanation and fixing tasks.",
       "evidence": "Tables 10-11 show large margins: Granite-8B-Code-Base achieves 26.4% avg on HumanEvalExplain vs 13.5% for StarCoder2-7B and 12.4% for CodeGemma-7B; 29.6% avg on HumanEvalFix vs 8.9% for StarCoder2-7B and 10.1% for CodeGemma-7B.",
       "supported": "strong"
     },
     {
       "claim": "Domain-specific code models are more suitable for cost- and performance-sensitive enterprise environments than larger general-purpose models.",
       "evidence": "Performance results show Granite outperforming some larger models on coding tasks (Section 6.1.1), but no cost analysis, latency measurement, or enterprise deployment evaluation is provided to support the enterprise suitability claim.",
       "supported": "weak"
     },
     {
       "claim": "Depth upscaling from the 20B model is effective for training the 34B model, with small initial performance drop that is quickly recovered.",
       "evidence": "Section 3 (34B description) and Figure 2 describe the depth upscaling approach. The claim that 'drop in performance compared to 20B model is pretty small' is stated but no quantitative evidence of the drop or recovery curve is provided in the paper.",
       "supported": "weak"
     },
     {
       "claim": "Instruction tuning consistently improves function calling performance, with +17.88% overall accuracy from Granite-8B-Code-Base to Granite-8B-Code-Instruct.",
       "evidence": "Figures 4 and 5 show BFCL results with clear improvement from base to instruct variants.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "All 46 authors are IBM Research employees evaluating IBM's Granite Code models, which are part of IBM's commercial WatsonX product line. The paper acknowledges WatsonX Code Assistant in the references. This is a textbook conflict of interest where the funder has a direct commercial stake in positive results."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Training data includes GitHub code and publicly available datasets. Evaluation benchmarks (HumanEval 2021, MBPP 2021, etc.) have solutions widely available on GitHub. No decontamination, overlap analysis, or temporal leakage discussion is provided despite this being a known and serious issue for code LLM evaluation."
     },
     {
       "flag": "No error bars or statistical tests",
       "detail": "All results are point estimates despite sampling-based evaluations. Numerous 'outperforms' claims are made by comparing raw numbers without any statistical significance testing. Small differences (e.g., 0.1% on HumanEvalSynthesize between Granite-20B and StarCoder2-15B) are treated as meaningful."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations section, no threats to validity, and no scope boundaries. For a paper with 46 authors and strong enterprise positioning claims, the complete absence of self-critical analysis is concerning."
     },
     {
       "flag": "Enterprise claims without enterprise evaluation",
       "detail": "The paper repeatedly claims models are 'optimized for enterprise software development workflows' and suitable for 'enterprise environments,' but all evaluation is on academic benchmarks. No real-world enterprise tasks, developer studies, or deployment metrics are provided."
     },
     {
       "flag": "Selective baseline presentation",
       "detail": "Some tables include different model sets, making cross-table comparison difficult. CodeGemma-2B is not included in all evaluations where Granite-3B appears. The Llama-3-70B Python generation issue (footnote in Table 3) is flagged but no investigation is described."
     }
   ],
   "cited_papers": [
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM that established data filtering and training methodology standards for code models."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Key baseline code LLM family from Meta, demonstrating code specialization through continued pretraining."
     },
     {
       "title": "OctoPack: Instruction Tuning Code Large Language Models",
       "authors": ["Niklas Muennighoff"],
       "year": 2023,
       "relevance": "Introduced HumanEvalPack benchmark for multilingual code generation/fixing/explanation and CommitPackFT used in Granite instruction tuning."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "State-of-the-art open code LLM and training dataset, key baseline for Granite Code evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "relevance": "Introduced HumanEval benchmark, the most widely used code generation evaluation, used extensively in this paper."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "relevance": "Introduced MBPP benchmark for Python code generation evaluation, used as key benchmark in this paper."
     },
     {
       "title": "MultiPL-E: A Scalable and Polyglot Approach to Benchmarking Neural Code Generation",
       "authors": ["Federico Cassano"],
       "year": 2023,
       "relevance": "Multilingual code generation benchmark covering 18 languages, used for comprehensive language evaluation."
     },
     {
       "title": "CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution",
       "authors": ["Alex Gu"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Code reasoning and execution benchmark testing input/output prediction, evaluating deeper code understanding."
     },
     {
       "title": "RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems",
       "authors": ["Tianyang Liu"],
       "year": 2023,
       "arxiv_id": "2306.03091",
       "relevance": "Repository-level code completion benchmark testing cross-file understanding, used for evaluating practical code completion."
     },
     {
       "title": "Lost in Translation: A Study of Bugs Introduced by Large Language Models while Translating Code",
       "authors": ["Rangeet Pan"],
       "year": 2024,
       "relevance": "CodeLingua benchmark for code translation quality, evaluating LLM capability in cross-language code tasks."
     },
     {
       "title": "RoCode: Robustness Evaluation of Code Generation Models",
       "authors": ["Shiqi Wang"],
       "year": 2022,
       "doi": "10.48550/arXiv.2212.10264",
       "relevance": "Robustness evaluation benchmark using perturbations on code generation, assessing model reliability under input variations."
     },
     {
       "title": "Berkeley Function Calling Leaderboard",
       "authors": ["Fanjia Yan"],
       "year": 2024,
       "relevance": "Function calling benchmark evaluating LLM ability to generate correct function calls, relevant to tool-use and agentic capabilities."
     },
     {
       "title": "Efficient Training of Language Models to Fill in the Middle",
       "authors": ["Mohammad Bavarian"],
       "year": 2022,
       "relevance": "Introduced FIM (Fill-in-the-Middle) training objective used in Granite Code models for code completion."
     }
   ]
 }