ai-research-survey

scan.json (26172B)

---

 {
   "paper": {
     "title": "Gorilla: Large Language Model Connected with Massive APIs",
     "authors": ["Shishir G. Patil", "Tianjun Zhang", "Xin Wang", "Joseph E. Gonzalez"],
     "year": 2023,
     "venue": "Neural Information Processing Systems",
     "arxiv_id": "2305.15334"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Gorilla, a finetuned LLaMA-7B model, surpasses GPT-4 on API call generation across TorchHub, HuggingFace, and TensorHub benchmarks while significantly reducing hallucination. Retriever-aware training enables adaptation to test-time documentation changes. The paper introduces APIBench with 1,645 APIs and 16,450 instruction-API pairs, and uses AST sub-tree matching for functional correctness evaluation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Gorilla's code, model, data, and demo are available at https://gorilla.cs.berkeley.edu'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code, model, data, and demo are available at gorilla.cs.berkeley.edu. The APIBench dataset is described as released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Training hyperparameters are listed in Table 4 (learning rate, batch size, etc.) and hardware is mentioned (8xA100 40G), but no requirements.txt, Dockerfile, or detailed dependency specification is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper points to a website but does not include commands or scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3 are point estimates (e.g., '59.13% accuracy') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Gorilla outperforms GPT-4 and other models based solely on comparing accuracy numbers. No statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '20.43% better than GPT-4 and 10.75% better than ChatGPT' and '83%' improvement over LLAMA (Section 4.1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the dataset sizes (10 instruction-API pairs per API, 90/10 or 80/20 train/test splits). No power analysis or sample size rationale."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multi-run results are reported. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against GPT-4, GPT-3.5-turbo, Claude, and LLaMA-7B across multiple retrieval settings (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 (gpt-4-0314), GPT-3.5-turbo, and Claude (claude-v1) were state-of-the-art models at the time of publication in 2023."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares Gorilla trained with vs. without retrieval, and tests each with different retrieval methods at inference time. Section 4.1 discusses the impact of retrieval during finetuning vs. evaluation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports overall accuracy, hallucination rate, and error rate as separate metrics (Table 1). Constraint-aware accuracy is reported as an additional metric (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via AST sub-tree matching."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'we have maintained a holdout test set on which we report our findings. The holdout test set was created by dividing the self-instruct dataset's instruction, API pairs into training and testing sets.' Appendix 8.2 specifies 90/10 for HuggingFace and 80/20 for Torch/Tensor Hub."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per API hub (TorchHub, HuggingFace, TensorHub) and per retrieval setting (0-shot, BM25, GPT-Index, Oracle) in Tables 1-3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 discusses hallucination examples from GPT-4 on HuggingFace (Figure 9), and Figure 1 shows failure cases for GPT-4 and Claude. The paper categorizes errors into hallucination vs. wrong API."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that adding a non-optimal retriever at test time can hurt performance: 'adding a non-optimal retriever at test time will sometime misguide the model and result in more errors' with specific degradation numbers (21.50% in TorchHub, 47.57% in HuggingFace)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims Gorilla 'surpasses the performance of GPT-4 on writing API calls' which is supported by Table 1. Claims about hallucination reduction and test-time adaptation are supported by Tables 1-2 and Figure 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about finetuning and retrieval-aware training improving performance. Table 2 provides controlled ablation comparing training with vs. without retrieval, which is an adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Large Language Model Connected with Massive APIs' but the evaluation only covers ML model hubs (TorchHub, HuggingFace, TensorHub). The paper acknowledges in Section 6 that they 'chose ML APIs' but the title and framing suggest broader applicability than tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example, the paper doesn't consider whether Gorilla's advantage is due to memorizing the specific API database used for both training and testing, or whether synthetic instruction generation introduces biases."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures AST sub-tree matching accuracy as a proxy for 'ability to use tools via API calls' but does not discuss this gap. AST matching checks syntactic correctness of API calls but doesn't verify whether the API call actually accomplishes the user's task. The paper acknowledges they have an execution system but 'that is not a focus of this paper.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model checkpoints are given: 'gpt-4-0314', 'gpt-3.5-turbo-0301', 'claude-v1', 'LLaMA-7B', and retriever 'text-davinci-003' (Section 4)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 provides full example prompts including the retrieval augmentation format. The paper specifies the exact concatenation template: '<user_prompt> Use this API documentation for reference: <retrieved_API_doc_JSON>'."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 lists training hyperparameters: learning rate 2e-5, batch size 64, 5 epochs, warmup ratio 0.03, weight decay 0, max seq length 2048."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Gorilla is a single-pass finetuned model that optionally has a retriever prepended to the input."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the full data collection pipeline: scraping model hubs, filtering criteria (top 20 per domain for HuggingFace, filtering models with poor documentation), converting to JSON format, and generating 10 instructions per API via GPT-4 self-instruct with 3 in-context examples."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations & Social Impacts' discusses limitations including the ML-domain focus and potential bias from ML APIs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section only discusses the ML domain focus and generic concerns about biased ML predictions. It does not address specific threats like the synthetic evaluation setup, potential data leakage, or the limited scope of AST-based evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions ML APIs as a limitation but does not explicitly state what the results do NOT show. No specific exclusions or non-claims are articulated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that code, model, data, and demo are available at gorilla.cs.berkeley.edu, and the dataset of 16,450 instruction-API pairs is released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection in detail: scraping model hubs, filtering criteria, converting model cards to JSON with specific fields, and generating instructions via GPT-4 self-instruct."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data sources are public API hubs (HuggingFace, TorchHub, TensorHub), which are standard public resources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 1,645 APIs scraped → converted to JSON → 10 instructions generated per API via self-instruct → 16,450 pairs → split into train/test. Filtering counts are given (925 HuggingFace, 626 TensorHub after filtering, 95 TorchHub)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states: 'This research is supported in part by gifts to UC Berkeley Sky Computing Lab from Astronomer, Google, IBM, Intel, Lacework, Microsoft, Nexla, Samsung SDS, Uber, and VMware.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: UC Berkeley and Microsoft Research. One author (Xin Wang) is from Microsoft Research."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft is both a funder (listed in acknowledgments) and has an author on the paper (Xin Wang, Microsoft Research). Microsoft has commercial interest in LLM tool use. Google, also a funder, competes in LLM space. The funders are not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. Given funding from multiple major tech companies and an author from Microsoft Research, a competing interests declaration is warranted but absent."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for GPT-4, GPT-3.5, Claude, or the base LLaMA model used for Gorilla."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses APIBench which is constructed from public model hubs. These API descriptions may appear in LLM training data, but no overlap analysis is performed. Additionally, the self-instruct data was generated by GPT-4, and GPT-4 is also a baseline — this creates potential contamination that is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The API documentation from HuggingFace, TorchHub, and TensorHub is publicly available and likely in GPT-4 and GPT-3.5 training data. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token consumption is reported for any model. The paper does not mention API costs for GPT-4/GPT-3.5 calls or inference time for Gorilla."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training hardware is mentioned (8xA100 40G) but total GPU hours, training time, or API costs for generating the self-instruct data are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results reported. All results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The hyperparameters in Table 4 appear fixed with no mention of search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the final configuration was selected or whether multiple configurations were tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons across the many model × retriever × dataset comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (Gorilla) against baselines without acknowledging author-evaluation bias. They control the benchmark, the evaluation metric, and the training pipeline."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Gorilla is a finetuned 7B model compared against much larger models (GPT-4) but the compute difference is not discussed. GPT-4 uses zero-shot inference while Gorilla was specifically finetuned on the target API data — this asymmetry is not addressed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "AST sub-tree matching is used as the evaluation metric but the paper does not discuss whether matching API function signatures actually measures the ability to correctly use APIs. The paper acknowledges they have execution capability but chose not to use it."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Gorilla is finetuned on the target API data while baselines are prompted zero-shot or with retrieval. This is a confound between finetuning-on-task vs. general capability that is not addressed — the comparison is between a specialist model and general-purpose models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses API documentation from public model hubs that existed before the training data cutoff of GPT-4 and GPT-3.5. This temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The oracle retriever setting provides the ground-truth API documentation to the model, which is a form of answer leakage in evaluation (acknowledged as an upper bound but not analyzed as feature leakage)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training and test data are generated from the same APIs via the same self-instruct process. Different instruction variants of the same API appear in both train and test. This non-independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used."
       }
     }
   },
   "claims": [
     {
       "claim": "Gorilla surpasses GPT-4 on writing API calls, achieving 20.43% better accuracy in zero-shot setting.",
       "evidence": "Table 1 shows Gorilla zero-shot accuracy of 59.13% on TorchHub vs GPT-4's 38.70%, 71.68% vs 19.80% on HuggingFace, and 83.79% vs 18.20% on TensorHub.",
       "supported": "moderate"
     },
     {
       "claim": "Gorilla substantially mitigates hallucination compared to other LLMs.",
       "evidence": "Table 1 shows Gorilla zero-shot hallucination rates of 6.98%, 10.95%, 5.40% across three hubs, compared to GPT-4's 36.55%, 37.16%, 78.65%.",
       "supported": "moderate"
     },
     {
       "claim": "Retriever-aware training enables Gorilla to adapt to test-time API documentation changes.",
       "evidence": "Figure 6 shows three qualitative examples where changing the retrieved document changes Gorilla's output API call accordingly. No quantitative evaluation of adaptation is provided.",
       "supported": "weak"
     },
     {
       "claim": "Finetuning with retrieval achieves 12.37% better results than without retrieval on TorchHub and 23.46% better on HuggingFace (with oracle retriever).",
       "evidence": "Table 2 compares Gorilla trained with vs. without oracle retriever, showing 67.20% vs 54.83% on TorchHub and 91.26% vs 45.58% on HuggingFace with oracle at inference.",
       "supported": "moderate"
     },
     {
       "claim": "Adding a non-optimal retriever at test time can hurt performance.",
       "evidence": "Section 4.1 reports performance drops when using BM25 or GPT-Index with a model finetuned without retrieval: 21.50% in TorchHub and 47.57% in HuggingFace.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Circular evaluation design",
       "detail": "Gorilla is finetuned on instruction-API pairs from the same APIBench dataset used for evaluation. The train/test split is from the same distribution (same APIs, same self-instruct process), giving it an inherent advantage over baselines that did not see this specific data format."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "Gorilla is finetuned specifically on API call data while baselines (GPT-4, Claude) are tested zero-shot or with simple retrieval. This tests domain-specific finetuning vs. general capability, not a fair model-to-model comparison."
     },
     {
       "flag": "No statistical rigor",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, or significance tests. Claims of superiority are based on raw number comparisons."
     },
     {
       "flag": "Qualitative-only evidence for key claim",
       "detail": "The test-time adaptation claim — a major selling point — is supported only by three cherry-picked qualitative examples (Figure 6) with no systematic quantitative evaluation."
     },
     {
       "flag": "Self-instruct evaluation bias",
       "detail": "GPT-4 was used to generate the instruction data, and GPT-4 is also a baseline. The synthetic instructions may be biased toward patterns GPT-4 generates well or poorly in ways that affect the comparison."
     },
     {
       "flag": "HuggingFace evaluation relaxed for baselines",
       "detail": "Section 4.1 notes: 'for HuggingFace, for all the models except Gorilla, we only check if they can provide the correct domain names.' This means the evaluation is easier for baselines on HuggingFace, yet Gorilla still uses full AST matching — making the comparison inconsistent."
     }
   ],
   "cited_papers": [
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Seminal work on training LLMs to use tools, directly relevant to agentic AI capabilities."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark for LLM code generation evaluation."
     },
     {
       "title": "Self-instruct: Aligning language model with self generated instructions",
       "authors": ["Yizhong Wang", "Yeganeh Kordi"],
       "year": 2022,
       "arxiv_id": "2212.10560",
       "relevance": "Core methodology used for generating Gorilla's training data; key technique in LLM training pipelines."
     },
     {
       "title": "HuggingGPT: Solving AI tasks with ChatGPT and its friends in HuggingFace",
       "authors": ["Yongliang Shen", "Kaitao Song"],
       "year": 2023,
       "arxiv_id": "2303.17580",
       "relevance": "Concurrent work on connecting LLMs to APIs for task solving via HuggingFace models."
     },
     {
       "title": "TaskMatrix.AI: Completing tasks by connecting foundation models with millions of APIs",
       "authors": ["Yaobo Liang", "Chenfei Wu"],
       "year": 2023,
       "arxiv_id": "2303.16434",
       "relevance": "Proposes connecting foundation models with millions of APIs for task completion, directly related to tool-use scaling."
     },
     {
       "title": "Reflexion: an autonomous agent with dynamic memory and self-reflection",
       "authors": ["Noah Shinn", "Beck Labash", "Ashwin Gopinath"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Agentic AI framework with self-reflection, relevant to LLM agent capabilities."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational work on combining reasoning and tool use in LLMs."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "relevance": "Benchmark work on LLM code generation at competition level."
     },
     {
       "title": "PAL: Program-aided language models",
       "authors": ["Luyu Gao", "Aman Madaan"],
       "year": 2022,
       "arxiv_id": "2211.10435",
       "relevance": "Demonstrates LLMs using code as a tool for reasoning, related to tool-use capabilities."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM relevant to code generation capabilities."
     }
   ]
 }