ai-research-survey

scan.json (28098B)

---

 {
   "paper": {
     "title": "LLMSecConfig: An LLM-Based Approach for Fixing Software Container Misconfigurations",
     "authors": ["Ziyang Ye", "Triet Huynh Minh Le", "M. Ali Babar"],
     "year": 2025,
     "venue": "IEEE Working Conference on Mining Software Repositories",
     "arxiv_id": "2502.02009",
     "doi": "10.1109/MSR66628.2025.00099"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLMSecConfig combines static analysis tools (Checkov) with LLMs and RAG to automatically repair Kubernetes security misconfigurations. Mistral Large 2 achieved a 94.3% repair pass rate on 1,000 real-world configurations, significantly outperforming GPT-4o-mini (40.2%). An ablation study found that source code context (90.3% PR) was more useful than documentation (65.2% PR), while combining all context types yielded the best results (94.3%).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section IX and contribution 3 state: 'we provide our complete implementation, including the collected dataset of Kubernetes misconfiguration, source code of the framework, and evaluation scripts at https://figshare.com/s/2a9be8ccfbec9d8ba199.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The same figshare link includes the collected dataset of Kubernetes misconfigurations, as stated in Section IX and contribution 3."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment specification is mentioned in the paper. The paper mentions Python (Checkov is pure Python) but provides no dependency versions or environment setup instructions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While source code and evaluation scripts are provided via figshare, the paper contains no step-by-step reproduction instructions. No README with commands or 'Reproducing Results' section is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables I and II report only point estimates (e.g., 94.3% PR, 40.2% PR) with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Mistral Large 2 'significantly outperforms' GPT-4o-mini and reports different performance across context types (Table II) without any statistical significance tests (no p-values, t-tests, or other tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results in Tables I and II provide baseline context for all comparisons (e.g., Mistral 94.3% vs GPT-4o-mini 40.2% PR; ablation from 88.0% base to 94.3% full context), allowing readers to assess the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset comprises 1,000 configurations from the top 1,000 ArtifactHub projects, but no justification is given for why 1,000 is sufficient, and no power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviations, variance across seeds, or repeat-run statistics are reported anywhere in Tables I or II."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o-mini serves as a baseline in RQ1 (Table I), and the ablation study in RQ3 (Table II) compares four context configurations as baselines against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Both Mistral Large 2 (2407) and GPT-4o-mini are 2024 models, making them contemporary at the time of writing. However, only two models are tested and both are on the cost-efficient end of the model spectrum."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Table II) is a systematic ablation study varying context types: Checkov output only, + source code, + Prisma documentation, and full context. This demonstrates which components contribute to performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Seven metrics are used: Pass Rate, Parse Success Rate, Average Pass Steps, AUCPRS, AUCAPSS, Security Improvement, and Average Introduced Errors (Section IV-D, Tables I-II)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via Checkov (SAT) scans. No human evaluation of repair quality, operational correctness, or semantic preservation is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper tuned hyperparameters (temperature=0.5 and max_retry=5) 'through experimentation' (Section V-A) but does not mention a separate validation set. The same 1,000 configurations appear to be used for both tuning and final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Fig. 8) provides per-security-policy-type breakdown of pass rates for both models across different misconfiguration categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 identifies and discusses categories of misconfigurations that are hard to correct, including complex privilege-related configurations for GPT-4o-mini (<50% PR) and advanced network policies for Mistral Large 2."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o-mini's poor performance (40.2% PR) is reported as a negative result. The ablation study shows Prisma documentation alone decreased performance (65.2% vs 88.0% base), which is a negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims '94% success rate' (Table I: 94.3%) and 'low rate of introducing new misconfigurations' (Table I: 0.024) are supported by the results. The abstract's claim about '1,000 real-world Kubernetes configurations' matches the experimental setup."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (RQ3) makes causal claims about which context types contribute to performance through controlled single-variable manipulation (adding/removing context sources while holding other variables constant). This is adequate for the ablation claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title says 'Software Container Misconfigurations' and abstract discusses 'Container Orchestrators (COs)' generally, but all experiments are on Kubernetes only. While Section IV-A notes Kubernetes as a 'case study,' the title and abstract do not adequately bound the scope to Kubernetes."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section VI-B discusses threats to validity but focuses on parameter settings and dataset representativeness. No alternative explanations are offered for the large performance gap between models, or why Prisma documentation hurts performance (described as 'noise' without deeper analysis)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Checkov pass rate and frames this as 'security configuration repair.' Passing Checkov checks is a proxy for actual security — a configuration could pass all Checkov checks and still be insecure via vectors Checkov doesn't cover. This gap is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "'mistral-large-2407' includes a version date, but 'GPT-4o-mini' is stated without a snapshot date or API version. Per schema criteria, marketing names without version identifiers do not count as specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C states 'More details about the prompt templates can be found in the Appendix at https://figshare.com/s/2a9be8ccfbec9d8ba199.' A repository link containing prompts meets the criterion."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A reports temperature=0.5, maximum parser retry time=10, and maximum retry value=5. Default values were used for other hyperparameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-phase pipeline is described in detail in Section III: SAT integration, context retrieval (RAG with three context sources), repair generation with inner/outer retry loops, and validation. Fig. 2 provides an architecture diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B documents the full data pipeline: ArtifactHub API → top 1,000 projects → Helm chart conversion → YAML parsing into individual sub-configurations → SAT scanning → retention of files with detected issues → 1,000 files."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B 'Threats to Validity' provides a dedicated subsection discussing internal and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Internal validity discusses 'potential sub-optimal configuration of our framework and baseline models' and their mitigation via sensitivity analysis. External validity notes results 'may not generalise to all CO platforms and security scenarios' with specific mention of Kubernetes focus."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While threats to validity mention generalizability concerns, the paper does not explicitly state what the results do NOT show. No clear enumeration of untested scenarios, excluded configuration types, or claims the authors are not making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section IX provides a figshare link with the complete dataset of Kubernetes misconfigurations, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B describes the collection: ArtifactHub API was used to identify top 1,000 most popular projects based on internal ranking, followed by Helm chart conversion and SAT scanning."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B describes the sample selection: top 1,000 most popular projects from ArtifactHub's internal ranking system, ensuring 'configurations widely used and representative of common industry practices.' However, this popularity-based selection could introduce bias toward well-maintained configurations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B documents each step: API access → top 1,000 projects → Helm chart to raw YAML conversion → individual sub-configuration isolation → SAT analysis → retention of misconfigured files → 1,000 files. Fig. 6 shows the distribution of misconfiguration types."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from CREST (Centre for Research on Engineering Software Technologies) and University of Adelaide. They do not evaluate their own product, so there is no product affiliation conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. The absence of a funding disclosure is a transparency gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosure, or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for either Mistral Large 2 or GPT-4o-mini. The models are evaluated on publicly available Kubernetes configurations that could have been in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The 1,000 configurations are from ArtifactHub, a public repository. Both models could have seen these configurations during training. This overlap is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ArtifactHub configurations are publicly available and likely predate both models' training cutoffs. No discussion of whether models have memorized common Kubernetes security patterns from training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated using SATs on Kubernetes configurations."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on configuration files."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite proposing a framework for production use and using API-based models with retry mechanisms (up to 5 retries × 10 parser retries), no inference costs, tokens consumed, API costs, or wall-clock times are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, GPU hours, API spend, or hardware specifications are mentioned despite running 1,000 configurations through multiple retry iterations."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. Temperature=0.5 introduces non-determinism but results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results appear to be single-run, with the retry mechanism being part of the method rather than repeated independent evaluations."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature was selected 'based on findings from prior research' (citation [44], which is about graph tasks) and max_retry was determined 'through experimentation,' but no search budget is reported — how many configurations were tried is unstated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of temperature=0.5 is justified by citing prior work on graph tasks [44], not configuration repair. Max_retry=5 was chosen 'through experimentation' without describing the selection process or showing results for other values."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the LLMSecConfig framework and evaluate it without acknowledging potential self-evaluation bias. No independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Mistral Large 2 and GPT-4o-mini likely have different computational costs, but no comparison at matched compute budgets is provided. The paper does not discuss whether the performance gap is due to model capability or compute differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Checkov pass rate as the primary measure of security but does not discuss whether passing Checkov checks actually means configurations are secure. Checkov's coverage of all security issues is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Both Mistral Large 2 and GPT-4o-mini are evaluated within the same LLMSecConfig framework with identical prompts, context retrieval, and retry mechanisms, controlling for the scaffold confound across model comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ArtifactHub configurations are publicly available and could have been in both models' training data. No discussion of temporal relationship between dataset creation and model training cutoffs."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The framework provides Checkov error messages, policy source code, and Prisma documentation as context. Whether this level of context would be available in real deployment (particularly the source code of checks) is not discussed as potential leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether configurations from the same ArtifactHub projects share structural similarities that could inflate results, or whether the top 1,000 projects represent independent samples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used despite the dataset being derived from publicly available sources."
       }
     }
   },
   "claims": [
     {
       "claim": "Mistral Large 2 achieves a 94.3% repair pass rate on 1,000 real-world Kubernetes configurations with 100% parse success rate.",
       "evidence": "Table I in Section V-A: Mistral Large 2 achieves 94.3% PR, 100% PSR, 3.06 APS, 0.986 security improvement, and 0.024 average introduced errors.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o-mini achieves only a 40.2% pass rate, significantly underperforming Mistral Large 2.",
       "evidence": "Table I in Section V-A: GPT-4o-mini achieves 40.2% PR, 99.8% PSR, 4.38 APS. No statistical significance test accompanies the 'significantly outperforms' language.",
       "supported": "moderate"
     },
     {
       "claim": "Source code context provides the most effective guidance for repairs (90.3% PR), while Prisma documentation alone decreases performance (65.2% PR).",
       "evidence": "Table II in Section V-C: ckv_out+code achieves 90.3% PR vs base ckv_out at 88.0% and ckv_out+prisma at 65.2%. Full context (94.3%) is best overall.",
       "supported": "moderate"
     },
     {
       "claim": "Both models maintain low error introduction rates (<0.03), demonstrating viability for production use.",
       "evidence": "Table I: GPT-4o-mini 0.029, Mistral Large 2 0.024 average introduced errors. However, production viability claim is unsupported — no human evaluation, cost analysis, or real-world deployment testing.",
       "supported": "weak"
     },
     {
       "claim": "Complex security contexts, especially privilege-related configurations, are consistently challenging for GPT-4o-mini with <50% pass rates.",
       "evidence": "Section V-B and Fig. 8a show per-category breakdown with GPT-4o-mini struggling on privilege-related policies. Mistral Large 2 shows better but still imperfect performance on advanced network policies (Fig. 8b).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Weak baseline selection",
       "detail": "Only two models are tested: Mistral Large 2 and GPT-4o-mini (OpenAI's cheapest/smallest model). No comparison with GPT-4o, GPT-4, Claude, or other competitive models. The baseline appears chosen to make the primary model look good."
     },
     {
       "flag": "No variance or repeat runs",
       "detail": "With temperature=0.5 introducing non-determinism and a retry mechanism, results could vary substantially across runs. All results appear to be from single runs with no error bars, standard deviations, or confidence intervals."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "Both models could have been trained on ArtifactHub configurations and Kubernetes security documentation. The models may have memorized common Kubernetes security patterns. This risk is entirely unacknowledged."
     },
     {
       "flag": "Hyperparameter justification from unrelated domain",
       "detail": "Temperature=0.5 is justified by citing [44] (Wang et al.), a paper about 'Reasoning with Large Language Models on Graph Tasks.' Configuration repair is a different task domain; the optimal temperature may differ."
     },
     {
       "flag": "No cost analysis for proposed production tool",
       "detail": "The paper advocates for production deployment (Section VI-A) but reports no API costs, latency, or compute requirements despite a retry mechanism that could multiply costs by up to 50x (5 retries × 10 parser retries per configuration)."
     },
     {
       "flag": "Proxy metric without validity discussion",
       "detail": "Checkov pass rate is used as the sole measure of 'security,' but Checkov cannot detect all security issues. A configuration passing Checkov checks may still be insecure, and this limitation is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu", "Z. Yang", "K. Wang", "L. Li", "X. Luo", "D. Lo", "J. Grundy", "H. Wang"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs in software engineering, directly relevant to the survey scope on LLM capabilities in SE tasks."
     },
     {
       "title": "Purple LLaMA CyberSecEval: A Secure Coding Benchmark for Language Models",
       "authors": ["M. Bhatt", "S. Chennabasappa"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "Benchmark for evaluating LLM security capabilities in code generation, relevant to LLM safety and code quality evaluation."
     },
     {
       "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models",
       "authors": ["M. Bhatt", "S. Chennabasappa"],
       "year": 2024,
       "arxiv_id": "2404.13161",
       "relevance": "Extended security evaluation benchmark for LLMs, relevant to assessing AI safety in code-related tasks."
     },
     {
       "title": "CyberSecEval 3: Advancing the Evaluation of Cybersecurity Risks and Capabilities in Large Language Models",
       "authors": ["S. Wan", "C. Nikolaidis", "D. Song"],
       "year": 2024,
       "arxiv_id": "2408.01605",
       "relevance": "Third iteration of cybersecurity evaluation for LLMs, relevant to evolving AI safety benchmarks."
     },
     {
       "title": "Deep Learning for Source Code Modeling and Generation: Models, Applications, and Challenges",
       "authors": ["T. H. M. Le", "H. Chen", "M. A. Babar"],
       "year": 2020,
       "relevance": "Survey on deep learning for code generation, a predecessor to the LLM-based code generation work in the survey scope."
     },
     {
       "title": "Retrieval-Augmented Generation for Large Language Models: A Survey",
       "authors": ["Y. Gao", "Y. Xiong", "X. Gao"],
       "year": 2023,
       "arxiv_id": "2312.10997",
       "relevance": "Survey of RAG techniques used in LLM systems, relevant to understanding augmentation methods for LLM-based tools."
     },
     {
       "title": "VulRepair: A T5-Based Automated Software Vulnerability Repair",
       "authors": ["M. Fu", "C. Tantithamthavorn", "T. Le", "V. Nguyen", "D. Phung"],
       "year": 2022,
       "relevance": "LLM-based approach for automated vulnerability repair in source code, closely related to LLM code repair capabilities."
     },
     {
       "title": "LineVul: A Transformer-Based Line-Level Vulnerability Prediction",
       "authors": ["M. Fu", "C. Tantithamthavorn"],
       "year": 2022,
       "relevance": "Transformer-based vulnerability detection at line level, relevant to AI-assisted code security analysis."
     },
     {
       "title": "KGSecConfig: A Knowledge Graph Based Approach for Secured Container Orchestrator Configuration",
       "authors": ["M. U. Haque", "M. M. Kholoosi", "M. A. Babar"],
       "year": 2022,
       "relevance": "Prior approach to container security configuration using knowledge graphs, predecessor to LLMSecConfig and relevant baseline for AI-assisted security."
     },
     {
       "title": "Data Quality for Software Vulnerability Datasets",
       "authors": ["R. Croft", "M. A. Babar", "M. M. Kholoosi"],
       "year": 2023,
       "relevance": "Study on data quality in vulnerability datasets, relevant to understanding data quality issues in AI security evaluations."
     }
   ]
 }