ai-research-survey

scan.json (21835B)

---

 {
   "paper": {
     "title": "Exploring Automatic Cryptographic API Misuse Detection in the Era of LLMs",
     "authors": ["Yifan Xia", "Zichen Xie", "Peiyu Liu", "Kangjie Lu", "Yan Liu", "Wenhai Wang", "Shouling Ji"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2407.16576"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We will open-source the refined benchmarks with detailed analysis for future research' — a promise of future release, not an actual release. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks used (CryptoAPI-Bench, MASC, ApacheCryptoAPI-Bench) are publicly available, but the paper's refined versions and the 11,940 LLM-generated reports are not released. They state they 'will release the misuse reports and analysis' — future tense."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The appendix mentions specific API model versions (gpt-3.5-turbo-1106, gpt-4-turbo-1106, Gemini-pro-1.0) and that open-source models used Hugging Face, with 'default settings for model hyper-parameters like temperature.' No requirements.txt, Dockerfile, or detailed environment setup is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology is described at a high level but lacks the specificity needed for exact reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (precision, recall, accuracy) with no confidence intervals or error bars despite the stochastic nature of LLMs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., GPT-4 outperforms SATs by 20.8%) but no statistical significance tests are applied to any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., '33.1% increase in FP alerts', '20.1% average accuracy increase', '20.8% increase in performance over the leading SATs', with full precision/recall/accuracy tables providing absolute values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of 5 queries per test case in the multi-query approach, nor for the selection of top 200 Java repositories in the usability study."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite running 5 queries per test case (acknowledging LLM stochasticity), no variance or standard deviation across these runs is reported. Only aggregated results are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three SOTA static analysis tools (CryptoGuard, CogniCryptSAST, SpotBugs) are compared against the LLMs in RQ2, with detailed accuracy, precision, and recall comparisons (Figure 4, Figure 7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CryptoGuard (2019) and CogniCryptSAST (2017) are the leading academic tools in this specific domain, and SpotBugs is widely used in industry. These are appropriate baselines given the niche area."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically evaluates components: unconstrained vs. task-aware settings, with and without the code & analysis validation mechanism, showing each component's contribution (Table 2, Table 3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Precision, recall, and accuracy are all reported, along with raw TP/FP/TN/FN counts in Tables 2 and 3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two authors with 5+ years of experience independently reviewed LLM outputs to classify TP/FP/FN, with cross-checking to minimize bias (Section 3.4). The usability study also involved developer feedback on reported misuses."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The usability study (RQ3) uses real-world GitHub repositories not part of the benchmarks, serving as an independent validation of the approach."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by complexity level (basic/advanced/mutation in Figure 3), by misuse category (Table 4), by file size impact, and by failure pattern categories with percentages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Three failure patterns are extensively analyzed: Erroneous Cryptographic Knowledge (41.3% of FPs), Code Semantics Misunderstanding (55.2% of FPs), and Hallucination/Denial-of-Service (3.4%). Sub-categories of code misunderstanding are further broken down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: GPT-4 performs worse than GPT-3.5 without validation (Finding 3 in RQ1), FP rates exceed 50% in unconstrained settings, performance degrades significantly with larger programs, and the validation mechanism sometimes removes true positives."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims ~90% detection rate (GPT-4 achieves 0.90 recall in Table 2), surpassing traditional methods (shown in Figure 4), and 63 real-world misuses with 46 acknowledged (Table 4 shows 46 accepted). All are supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about the validation mechanism improving performance are supported by controlled ablation (with/without validation). Claims about task-aware settings improving results are similarly supported by systematic comparison. The ablation design is adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Exploring Automatic Cryptographic API Misuse Detection in the Era of LLMs' is appropriately scoped, but the paper's generalization discussion (Section 5.2.3) is brief and generic: 'our findings may still not generalize to other datasets or languages.' The abstract claims 'surpassing traditional methods' without bounding to the specific benchmarks tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.1 discusses data leakage as an alternative explanation for LLM performance, providing statistical analysis (0.3% overlap) to argue against it. The paper also discusses how misleading benchmark metadata could inflate results, leading them to rename test cases."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The appendix specifies exact versions: gpt-3.5-turbo-1106, gpt-4-turbo-1106, Gemini-pro-1.0, CodeLlama-34B-Instruct, DeepSeek-Coder-33b-Instruct (Section 7.1.2)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Figures 5 and 6 of the appendix, showing both the detection and validation prompts with system roles, basic prompts, setting-specific components, and formatting instructions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'All experiments maintain the default settings for model hyper-parameters like temperature' but does not specify what those defaults are. Default temperature varies by API version and is not always documented."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-query approach (5 queries per test case) and code & analysis validation mechanism are described in detail in Section 3.3.2, with the workflow illustrated in Figure 1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.2 documents benchmark refining steps: removing inapplicable cases (context-insensitive, obsolete, redundant), addressing ground truth leakage by standardizing naming, with specific counts (154 GTMs in manually-crafted, 53 GTMs in real-world after filtering)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.2 discusses limitations and Section 5.2 is dedicated to 'Threat of validity' covering data leakage, manual analysis reliability, and generalization concerns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: data leakage from cryptographic benchmarks into LLM training data (with statistical analysis), bias in manual TP/FP classification, and limitations of Java/Python-only evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2.3 only states generically 'our findings may still not generalize to other datasets or languages.' No specific boundaries are drawn about what the results do NOT show (e.g., does not show LLMs can replace SATs in production, does not demonstrate reliability for safety-critical deployment)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 11,940 LLM-generated reports are not released. The paper promises future release but no data is currently available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is well described: benchmark selection and refining (Section 3.2), LLM querying procedure with 5 queries per test case (Section 3.3.2), and repository selection for usability study (Section 4.4.1, with criteria in Appendix Section 5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The developer feedback in the usability study was from maintainers of existing open-source projects contacted through GitHub advisories/issues, not recruited participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: benchmark selection → refining (with counts: 154 GTMs manually-crafted, 53 real-world) → LLM querying (5x per case) → validation mechanism → manual analysis by two reviewers with cross-checking."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Zhejiang University, University of Minnesota Twin Cities, and Ant Group. Ant Group is a technology company but is not a producer of the evaluated LLMs."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure is itself a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 lists training data cutoff dates for all models: GPT-3.5-turbo (Sep 2021), GPT-4-turbo (Apr 2023), Gemini-1.0-pro (July 2023), CodeLlama (July 2023), DeepSeek-Coder (May 2023)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.1 extensively discusses data leakage concerns, analyzing whether benchmark ground truths could appear in training data. They find only 0.3% overlap for CryptoAPI-Bench and zero for ApacheCryptoAPI-Bench. They also address ground truth leakage from test case naming (Section 3.2.2)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper actively addresses contamination by renaming test cases to prevent metadata leakage (Section 3.2.2) and argues that the presence of numerous failure cases among LLMs makes memorization-based success unlikely (Section 5.2.1)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. The developer feedback is from open-source maintainers responding to bug reports, not a structured human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper mentions ethical considerations in anonymizing non-public reports (Section 4.4) but this does not constitute a human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API spend, or per-query cost is reported despite evaluating 11,940 LLM-generated reports across 5 models with multiple queries per test case."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours for open-source models, or API spend is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs' false positive rates exceed 50% even for GPT-4 in unconstrained settings on manually-crafted benchmarks.",
       "evidence": "Table 2 shows GPT-4 unconstrained without validation: precision 0.48, with 89 FP out of 172 total alerts.",
       "supported": "strong"
     },
     {
       "claim": "The optimized approach (task-aware + validation) achieves nearly 90% detection rate, surpassing traditional methods.",
       "evidence": "Table 2 shows GPT-4 TA w/V: recall 0.90, precision 0.87, accuracy 0.84. Figure 4 shows this exceeds all SATs.",
       "supported": "strong"
     },
     {
       "claim": "63 real-world cryptographic misuses were discovered, with 46 acknowledged by developers.",
       "evidence": "Table 4 shows 37 Java + 29 Python = 66 reported (after excluding 10 manual FPs), with 26+20=46 accepted.",
       "supported": "strong"
     },
     {
       "claim": "The code & analysis validation mechanism provides an average 20.1% accuracy increase.",
       "evidence": "Comparison of w/V vs w/o V rows in Tables 2 and 3 across all models supports this claim.",
       "supported": "moderate"
     },
     {
       "claim": "Task-aware setting generates 18.9% more TP alerts on average compared to unconstrained setting.",
       "evidence": "Finding 2 in Section 4.2.1 states this with range from 10.3% (GPT-3.5) to 32.5% (GPT-4), verifiable from Table 2.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "This paper systematically evaluates 5 LLMs for cryptographic API misuse detection using refined benchmarks. LLMs exhibit high false positive rates (>50%) in unconstrained settings, but task-aware scoping combined with a self-validation mechanism raises GPT-4's accuracy to 84-88%, surpassing SOTA static analysis tools. Three failure patterns are identified: erroneous cryptographic knowledge (41.3% of FPs), code semantics misunderstanding (55.2%), and hallucination (3.4%). A usability study on real GitHub repositories discovered 63 cryptographic misuses, with 46 acknowledged by developers and a 100% acceptance rate for Java findings.",
   "red_flags": [
     {
       "flag": "No variance reported despite stochastic methods",
       "detail": "The paper acknowledges LLM stochasticity and runs 5 queries per test case, yet reports only aggregated point estimates without any variance, standard deviation, or confidence intervals across runs."
     },
     {
       "flag": "No statistical significance tests for comparisons",
       "detail": "All comparative claims (LLMs vs SATs, with/without validation, task-aware vs unconstrained) are based on raw metric differences without any statistical tests."
     },
     {
       "flag": "No cost reporting",
       "detail": "11,940 LLM reports were generated across 5 models including GPT-4, yet no API costs, GPU hours, or computational budget is mentioned, making practical replicability unclear."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Models for Code Analysis: Do LLMs Really Do Their Job?",
       "authors": ["C. Fang", "N. Miao", "S. Srivastav"],
       "year": 2023,
       "arxiv_id": "2310.12357",
       "relevance": "Directly evaluates LLM capabilities for code analysis tasks, relevant to understanding LLM reliability."
     },
     {
       "title": "How ChatGPT is Solving Vulnerability Management Problem",
       "authors": ["P. Liu", "J. Liu", "L. Fu"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT for security vulnerability management, relevant to LLM-based security analysis."
     },
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 Bugs for $0.42 Each Using ChatGPT",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "Uses LLMs for automated program repair with cost analysis, relevant to LLM-based code quality tools."
     },
     {
       "title": "Enhancing Static Analysis for Practical Bug Detection: An LLM-Integrated Approach",
       "authors": ["H. Li", "Y. Hao", "Y. Zhai", "Z. Qian"],
       "year": 2024,
       "relevance": "Integrates LLMs with static analysis for bug detection, directly relevant to hybrid LLM+traditional tool approaches."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "relevance": "EvalPlus benchmark for evaluating LLM code generation correctness, relevant to LLM evaluation methodology."
     },
     {
       "title": "A Survey of Large Language Models",
       "authors": ["W. X. Zhao", "K. Zhou", "J. Li"],
       "year": 2023,
       "arxiv_id": "2303.18223",
       "relevance": "Comprehensive survey of LLMs covering hallucination and reliability issues central to this paper's concerns."
     },
     {
       "title": "A Systematic Evaluation of Large Language Models of Code",
       "authors": ["F. F. Xu", "U. Alon", "G. Neubig", "V. J. Hellendoorn"],
       "year": 2022,
       "relevance": "Systematic evaluation of code LLMs including temperature effects, relevant to LLM evaluation methodology."
     },
     {
       "title": "Why Crypto-Detectors Fail: A Systematic Evaluation of Cryptographic Misuse Detection Techniques",
       "authors": ["A. S. Ami", "N. Cooper", "K. Kafle"],
       "year": 2022,
       "relevance": "Systematic evaluation of cryptographic misuse detection tools and their failure modes, directly relevant baseline work."
     },
     {
       "title": "Can Large Language Models Reason About Program Invariants?",
       "authors": ["K. Pei", "D. Bieber", "K. Shi", "C. Sutton", "P. Yin"],
       "year": 2023,
       "relevance": "Evaluates LLM reasoning about program properties, relevant to understanding LLM code comprehension capabilities."
     }
   ]
 }