ai-research-survey

scan.json (24697B)

---

 {
   "paper": {
     "title": "Can LLM Replace Stack Overflow? A Study on Robustness and Reliability of Large Language Model Code Generation",
     "authors": ["Li Zhong", "Zilong Wang"],
     "year": 2023,
     "venue": "AAAI 2024",
     "arxiv_id": "2308.10335"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/FloridSleeves/RobustAPI. The dataset and evaluator are stated as open-sourced."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset of 1208 Stack Overflow questions is released as part of the GitHub repository. The paper states 'We open-source our dataset and evaluator on GitHub.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency versions are mentioned in the paper. There is no 'Environment Setup' section with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While the code is released, no README with commands to run or a 'Reproducing Results' section is described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '62.97% misuse rate') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'GPT-4 actually has a higher API misuse rate than GPT-3.5') but provides no statistical significance tests. All comparisons are made by directly comparing percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. Raw percentages are given but without standardized effect size measures or baseline context beyond the raw numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset size of 1208 questions is stated but not justified with a power analysis or rationale for why this number is sufficient. The selection of 18 APIs is also not statistically justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs (Pass@1 unless specified). No standard deviations, variance across runs, or spread measures are reported. The temperature experiment (Table 5) shows different temperature settings but these are different conditions, not repeated runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 4 main LLMs (GPT-3.5, GPT-4, Llama-2, Vicuna-1.5) plus DeepSeek-Coder variants against each other. While there is no non-LLM baseline, the multi-model comparison across settings serves as a baseline comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models evaluated (GPT-3.5, GPT-4, Llama-2, Vicuna-1.5, DeepSeek-Coder) were all contemporary and state-of-the-art at the time of the study (2023)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation-style experiments: zero-shot vs. one-shot-irrelevant vs. one-shot-relevant settings, temperature variations (Table 5), and API rules vs. one-shot examples (Table 6). These systematically vary components to measure their effect."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are defined and reported: API Misuse Rate, Compilation Rate, and Overall API Misuse Percentage. Pass@k results are also reported (Table 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation is entirely automated via AST-based static analysis. No human evaluation of the generated code quality or API misuse detection accuracy is reported. Given that the paper makes claims about code reliability for developers, human evaluation of the outputs would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The one-shot demonstration examples are stated to not be present in the testing dataset: 'The question in the demo example is not present in the testing dataset.' The entire 1208 questions serve as the test set with separate demonstration examples."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides a per-API breakdown of misuse rates for each model across all experiment settings. Table 1 shows the domain-level breakdown of the dataset."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The 'Error Analysis' section and 'Case Study: API Misuse in GPT-3.5' section discuss specific failure cases, showing concrete code examples where GPT-3.5 generates API misuse under different settings."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: irrelevant shots increase misuse rate rather than decreasing it (Finding 3), API usage rules in prompts do not help reduce misuse (Table 6, Finding 5), and relevant shots do not help Llama."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'even GPT-4 has 62% of the generated code that contains API misuses' which is supported by Table 2 (GPT-4 zero-shot Overall Misuse = 62.09%). Other abstract claims about the dataset and evaluation framework are supported in the methodology and results sections."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'the irrelevant shot provided to the large language models actually encourages the models to give a lengthy code solution, which increases the chance of API misuse' without adequate causal evidence. This is speculation about mechanisms without controlled testing of this specific hypothesis."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title asks 'Can LLM Replace Stack Overflow?' which implies general conclusions, but results are limited to 18 Java APIs and 4-6 LLMs. The Discussion section acknowledges the Java focus ('Extend to Other Language') but the title and framing significantly overreach the tested scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations. For example, the finding that GPT-4 has higher misuse than GPT-3.5 is noted as 'counter-intuition' but not deeply explored. No threats-to-validity section exists, and possible confounds (e.g., training data composition, prompt sensitivity) are not systematically addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to models as 'GPT-3.5', 'GPT-4', 'Llama-2', 'Vicuna-1.5' without specifying exact API versions or snapshot dates (e.g., no 'gpt-4-0613'). DeepSeek-Coder variants are specified by size (6.7b-base, 6.7b-instruct) but not by exact version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the prompt structure in natural language ('we start with the task introduction and the required response format') and mentions it is adapted from Patil et al. 2023, but the actual full prompt text is not provided in the paper or appendix. Only a template description is given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We use the default hyper-parameter settings of each model without further extensive hyper-parameter tuning' but does not specify what those defaults are (temperature, top-p, max_tokens). Table 5 shows temperature experiments but only for GPT-3.5 one-shot-relevant."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is a straightforward prompt-response setup with no tool use, retry logic, or multi-turn interaction."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data collection pipeline is described: starting from ExampleCheck dataset, selecting 18 APIs, crawling Stack Overflow questions, filtering for those with online answers containing API misuse, resulting in 1208 questions. The filtering criteria and conversion to JSON format are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The Discussion section briefly mentions extending to other languages but does not constitute a substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. Potential issues such as the accuracy of the AST-based checker, the representativeness of the 18 selected APIs, or the impact of prompt design on results are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The Discussion mentions extending to other languages but does not bound the claims to Java or to the specific API set tested. No explicit statements about what claims the authors are NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is open-sourced on GitHub (https://github.com/FloridSleeves/RobustAPI), allowing independent verification of the questions, API usage rules, and evaluation results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described in the 'Data Collection' section: based on ExampleCheck dataset, 18 popular Java APIs selected, questions crawled from Stack Overflow, filtered to keep only questions with answers containing API misuse, yielding 1208 questions."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data source is a standard benchmark constructed from Stack Overflow questions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from ExampleCheck to final dataset is documented: API selection from ExampleCheck -> Stack Overflow crawling -> filtering for questions with answers containing API misuse -> JSON conversion -> prompt generation. The evaluator pipeline (AST extraction -> call sequence comparison -> misuse detection) is also documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The Acknowledgments section only thanks AAAI reviewers and chairs without mentioning any funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are disclosed: both authors are from University of California, San Diego. They are not affiliated with any of the LLM companies being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure statement means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models. This is relevant because the Stack Overflow questions in the benchmark could have been in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss potential overlap between the Stack Overflow questions in ROBUSTAPI and the training data of the evaluated LLMs. Stack Overflow content is commonly used in LLM training data, making this a significant omission."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is constructed from Stack Overflow questions that were available online well before the training cutoffs of GPT-3.5/4, Llama-2, etc. No discussion of contamination risk is provided despite the high likelihood that these models were trained on Stack Overflow data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example costs are reported. Only execution time for the static analysis checker is provided (Table 3), not the LLM inference cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is stated. Table 3 only shows the static analysis execution time (~6-7 minutes), not the LLM query costs."
       }
     }
   },
   "claims": [
     {
       "claim": "Even GPT-4 has 62% of generated code containing API misuses in zero-shot setting.",
       "evidence": "Table 2 shows GPT-4 zero-shot Overall API Misuse Percentage of 62.09% and API Misuse Rate of 68.81% among compilable code.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 has a higher API misuse rate than GPT-3.5, despite being promoted as more advanced.",
       "evidence": "Table 2 shows GPT-4 zero-shot misuse rate of 68.81% vs. GPT-3.5 at 62.97%. However, no significance test is performed on this comparison.",
       "supported": "moderate"
     },
     {
       "claim": "Irrelevant one-shot examples do not reduce API misuse rate but trigger more valid answers.",
       "evidence": "Figure 3 and Table 2 show that one-shot-irrelevant setting increases compilation rate but generally maintains or increases misuse rate across models (Finding 2, Finding 3).",
       "supported": "moderate"
     },
     {
       "claim": "Relevant one-shot examples with correct API usage significantly reduce misuse rate for GPT-3.5, GPT-4, and Vicuna.",
       "evidence": "Table 2 shows GPT-3.5 misuse rate drops from 62.97% (zero-shot) to 38.56% (one-shot-relevant), GPT-4 from 68.81% to 54.40%, Vicuna from 45.66% to 42.53%.",
       "supported": "moderate"
     },
     {
       "claim": "Code-specialized LLMs (DeepSeek-Coder) generate more compilable samples but do not significantly improve API misuse rate.",
       "evidence": "Table 2 shows DeepSeek-Coder-6.7b-instruct achieves 96.61% compilation rate (one-shot-irrelevant) but 59.04% misuse rate, comparable to other models.",
       "supported": "moderate"
     },
     {
       "claim": "Changing temperature or replacing one-shot examples with API rules does not significantly affect API misuse rate.",
       "evidence": "Table 5 shows similar misuse rates across temperatures (38.56% at T=0, 39.77% at T=0.5, 39.06% at T=1.0). Table 6 shows API rules actually increase misuse rate to 65.01% vs 38.56% for one-shot-relevant.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces ROBUSTAPI, a benchmark of 1208 Stack Overflow questions across 18 Java APIs to evaluate API misuse in LLM-generated code. All evaluated models (GPT-3.5, GPT-4, Llama-2, Vicuna-1.5, DeepSeek-Coder) exhibit high rates of API misuse (38-70% among compilable code), even when generating syntactically correct and functionally adequate code. Providing one-shot examples with correct API usage reduces misuse rates for some models (GPT-3.5 drops from 63% to 39%), while irrelevant examples or explicit API rules are ineffective. Counter-intuitively, GPT-4 shows higher API misuse rates than GPT-3.5 despite being considered more capable.",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "The benchmark is constructed from Stack Overflow questions that were publicly available before the training data cutoffs of all evaluated models. LLMs are commonly trained on Stack Overflow data. The paper does not address whether the models may have seen these questions during training, which could mean the results reflect memorization rather than generalization."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims between models are based on raw percentage comparisons with no significance tests. With 1208 questions, differences of a few percentage points may not be statistically significant, yet the paper draws conclusions from them (e.g., GPT-4 vs GPT-3.5 comparison)."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "Results appear to be from single runs with no confidence intervals, error bars, or variance across runs reported. Stochastic LLM outputs could produce different results on repeated runs."
     },
     {
       "flag": "Overclaimed title and framing",
       "detail": "The title 'Can LLM Replace Stack Overflow?' implies a broad conclusion, but the study only examines 18 Java APIs for API misuse patterns. It does not address other dimensions of Stack Overflow utility (debugging, conceptual explanations, architecture decisions, etc.)."
     },
     {
       "flag": "Missing limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. The accuracy of the AST-based checker, representativeness of selected APIs, and prompt design sensitivity are not discussed as potential threats."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "Exact model versions (API snapshots, dates) for GPT-3.5 and GPT-4 are not specified. Given that model behavior changes across versions, results may not be reproducible."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation evaluation, a foundational benchmark in the field."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Proposes real-world software engineering benchmark using GitHub issues, directly relevant to evaluating LLM coding capabilities."
     },
     {
       "title": "Asleep at the keyboard? assessing the security of github copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan"],
       "year": 2022,
       "relevance": "Evaluates security of LLM-generated code (Copilot), finding ~40% of code is vulnerable, relevant to code quality assessment."
     },
     {
       "title": "Do users write more insecure code with AI assistants?",
       "authors": ["Neil Perry", "Megha Srivastava", "Deepak Kumar", "Dan Boneh"],
       "year": 2022,
       "arxiv_id": "2211.03622",
       "relevance": "Large-scale user study on security implications of AI code assistants, directly relevant to AI coding productivity and safety."
     },
     {
       "title": "Lost at c: A user study on the security implications of large language model code assistants",
       "authors": ["Gustavo Sandoval", "Hammond Pearce", "Ton Nys"],
       "year": 2023,
       "arxiv_id": "2208.09727",
       "relevance": "User study assessing security of low-level code generated by AI assistants, complementary study to AI code quality research."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Rigorous evaluation of ChatGPT code generation correctness, enlarging HumanEval with higher-coverage test cases."
     },
     {
       "title": "Large Language Models and Simple, Stupid Bugs",
       "authors": ["Kevin Jesse", "Toufique Ahmed", "Premkumar T. Devanbu"],
       "year": 2023,
       "arxiv_id": "2303.11455",
       "relevance": "Studies simple bugs in LLM-generated code, showing AI assistants can introduce hard-to-detect bugs."
     },
     {
       "title": "Assessing the quality of GitHub copilot's code generation",
       "authors": ["Burak Yetistiren", "Isik Ozsoy", "Eray Tuzun"],
       "year": 2022,
       "relevance": "Assesses Copilot code quality across compilation correctness, functional correctness, and code efficiency."
     },
     {
       "title": "Gorilla: Large language model connected with massive apis",
       "authors": ["Shishir G. Patil", "Tianjun Zhang", "Xin Wang", "Joseph E. Gonzalez"],
       "year": 2023,
       "arxiv_id": "2305.15334",
       "relevance": "Addresses LLMs connected to APIs, relevant to understanding API usage and hallucination in code generation."
     },
     {
       "title": "A systematic evaluation of large language models of code",
       "authors": ["Frank F. Xu", "Uri Alon", "Graham Neubig", "Vincent J. Hellendoorn"],
       "year": 2022,
       "relevance": "Systematic evaluation of code LLMs, relevant to the broader evaluation methodology for AI code generation."
     }
   ]
 }