ai-research-survey

scan.json (25712B)

---

 {
   "paper": {
     "title": "CloudFix: Automated Policy Repair for Cloud Access Control Policies Using Large Language Models",
     "authors": [
       "Bethel Hall",
       "Owen Ungaro",
       "William Eiers"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2512.09957"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section IX states 'The policy dataset and accompanying source code used in this study are publicly available at https://github.com/bethelhall/fixmypolicy.' A GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section IX states the policy dataset is publicly available at the same GitHub URL. Section IV also states 'we curate real-world AWS IAM policies from the AWS re:Post forum and make them available to the research community.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section V-A describes hardware (Rocky Linux 9.3, NVIDIA L40S GPU, etc.) and states models were loaded via HuggingFace transformers. However, there are no requirements.txt, Dockerfile, or detailed library version listings sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the source code and dataset are released, the paper does not describe step-by-step reproduction instructions, a README with commands to run, or specific scripts to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables II-V report point estimates of accuracy percentages and counts without confidence intervals or error bars. No uncertainty quantification is provided for any of the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table II reports a two-tailed t-test comparing mean repair accuracies between baseline and Fault Localization, with p-values for each request size (p<0.001 for sizes 10, 20, 30; p=0.056 for size 50)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table II reports the difference in accuracy percentage points between baseline and Fault Localization (e.g., +17.55pp for size 10, +7.29pp for size 20), alongside the baseline values, providing enough context to judge magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset of 282 policies is described but not justified via power analysis or any explanation of why this sample size is sufficient. The RQ5 subset of only 10 policies is also not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for any experimental results. Tables show only means (e.g., 'Avg. Total Time') and counts without any indication of variability across policies."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares its Fault Localization guided approach against a baseline approach (simple prompt without fault localization). Section V-C describes both approaches and explicitly notes the baseline 'serves as an ablation to isolate the impact of Fault Localization.'"
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baseline is the authors' own simplified prompt (no Fault Localization). No comparison is made against prior symbolic policy repair tools (e.g., Eiers et al. 2023, D'Antoni et al. 2024) or other LLM-based repair approaches. The related work section discusses these but they are not experimentally compared."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The baseline vs. Fault Localization comparison functions as an ablation study, as stated in Section V-C: 'The baseline serves as an ablation to isolate the impact of Fault Localization.' This is a controlled single-variable manipulation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D defines three evaluation metrics: repair accuracy (percentage of correctly classified requests), iteration count, and total repair time. Tables III and IV report all three."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "There is no human evaluation of repair quality. Evaluation is entirely automated via SMT solver verification. Given that the repairs are access control policies, a human evaluation of repair quality or security implications would add value."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 explicitly tests generalization to held-out requests: 'we repaired each policy using both approaches, then tested generalizability by evaluating the repaired policies on 15 held-out variant requests not provided during the repair process.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table III provides per-request-size breakdowns. Table V provides per-model breakdowns. Results are reported across four request sizes (10, 20, 30, 50) and four models, with three accuracy categories (100%, 80-99%, <80%)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses failure modes in detail: 'condition omission (e.g., dropping IpAddress constraints), entity hallucination (substituting arn:aws:iam::123456789012:root for the actual principal), and syntactic degeneration where 46% of policies showed no improvement due to repeated invalid JSON generation until token exhaustion.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that at request size 50, the Fault Localization improvement is not statistically significant (p=0.056). It also reports poor performance of DeepSeek-Coder and Granite (75%+ failed repairs) and notes 'accuracy degradation at larger request sizes' where Fault Localization only generated 7 complete repairs."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'CloudFix improves repair accuracy over a baseline implementation across varying request sizes.' This is supported by Table II/III showing statistically significant improvements for request sizes 10, 20, and 30, though the abstract does not mention the non-significant result at size 50."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about Fault Localization improving repair performance. This is supported by a controlled ablation design where the only variable changed between baseline and FL-guided approaches is the inclusion of the fault localization report, with other factors held constant (same prompts, models, configurations)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Cloud Access Control Policies' but experiments are only on AWS IAM policies. The abstract claims 'enabling efficient and automated repair of cloud access control policies' without bounding to AWS. Section II-A acknowledges 'the techniques we present in this work are not specific to the AWS IAM policy language and can be applied to any policy language' but provides no evidence for other policy languages."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section VI discusses some limitations but does not discuss alternative explanations for the observed improvements. For example, the improvement could be due to the additional context length rather than the fault localization information specifically, or CodeLlama's dominance could reflect training data contamination with AWS policy formats. These alternatives are not considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists model names such as 'CodeLLaMA-7B-Instruct', 'Granite-3.3-8B-Instruct', 'DeepSeek-Coder-7B-Instruct', and 'Llama3.2-3B-Instruct'. For RQ5, 'GPT-5, accessed via its web interface' is mentioned without any version or snapshot date. The open-source models have size identifiers but no specific checkpoint versions or dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows a detailed example of the Fault Localization guided prompt. Section III-C states 'The complete prompts used in our experiments are provided in the accompanying artifact repository (see Section IX).' The prompt structure is also fully described."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report temperature, top-p, max tokens, or other generation hyperparameters. Section V-B states 'We maintain similar prompts and model hyperparameters for all models used' but does not list the actual values."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III provides a detailed description of the agentic scaffolding: the iterative repair loop (Alg. 1), Goal Validator (Alg. 2), Fault Localizer (Alg. 3), Prompt Generator, and Repair Synthesizer. The maximum iteration threshold, feedback mechanism, and workflow are all documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes the data pipeline: scraping 9,968 IAM-related posts, scanning for JSON structures with IAM policy keys, validating with Quacky, applying syntactic repair to enforce IAM conventions, resulting in 282 faulty policies and 45 community-accepted repairs. Section IV-B documents the request generation process (Alg. 4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI is titled 'Discussion and Threat to Validity' and contains substantive discussion of limitations including dataset representativeness, performance degradation at scale, and hardware constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses specific threats: 'Our dataset relies on policies from AWS re:Post with algorithmically generated requests, which may not fully capture real-world complexity.' It also notes 'accuracy degraded as request size grew to 50' and 'We restricted evaluation to smaller open-source models due to hardware constraints.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions some limitations in Section VI, it does not explicitly state scope boundaries about what the results do NOT show. It does not clearly say the results are limited to AWS IAM policies, small open-source models, or synthetically generated requests. The claims remain broad relative to the tested setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section IX states the dataset is publicly available at the GitHub repository. The 282 policies and generated request sets can be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes data collection: scraping 9,968 IAM-related posts from AWS re:Post, extracting JSON structures, validating with Quacky, and applying syntactic repair to obtain 282 policies. Table I provides summary statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The dataset consists of access control policies scraped from public forum posts. This is not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV documents the pipeline: scraping posts (9,968) → detecting JSON with IAM keys → validating with Quacky → syntactic repair → 282 faulty policies and 45 ground truth repairs. Request generation is also documented (Alg. 4) with parameters for allow/deny ratios and misclassification rates."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are listed with their affiliation at Stevens Institute of Technology, Department of Computer Science. One author (William Eiers) has prior work with the Quacky tool used in the framework, which is disclosed through co-authorship on the cited references."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. The absence of a funding disclosure does not mean the work is unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple LLMs on AWS IAM policies but does not state the training data cutoff for any of the models used (CodeLlama, Llama 3.2, DeepSeek-Coder, Granite, or GPT-5). AWS IAM policy examples are widely available online and could be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The policies are scraped from public AWS re:Post forums, which are publicly accessible on the internet. The LLMs used could have been trained on these same posts. No discussion of potential train/test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The AWS re:Post forum posts containing the policies are publicly available online and were posted before the training cutoff of any of the models used. The paper does not address whether the models may have seen these policies during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study. It is a benchmark evaluation of an automated tool."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available forum posts and automated evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table IV reports average total repair time in seconds (e.g., 136.5s for FL at request size 10, up to 1306.9s at size 50), broken down into SMT time and LLM time, along with average iteration counts. This provides wall-clock time cost per policy repair."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A states: 'Each job was allocated a single NVIDIA L40S GPU with 46 GB of memory, two Intel(R) Xeon(R) Platinum 8562Y+ CPU cores, and 256 GB of system RAM.' Table IV reports timing across all experiments. However, total GPU hours for the full experimental campaign are not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "CloudFix with Fault Localization achieves 84.0% complete repair rate on 10-request policies compared to 48.2% for baseline.",
       "evidence": "Table III shows 237/282 (84.0%) complete repairs for FL vs 136/282 (48.2%) for baseline at request size 10. Table II shows p<0.001 for this comparison.",
       "supported": "strong"
     },
     {
       "claim": "Fault Localization maintains 54.3% complete repair rate on 30-request policies while baseline drops to 22.3%.",
       "evidence": "Table III shows 153/282 (54.3%) for FL vs 63/282 (22.3%) for baseline at request size 30. Table II shows p<0.001.",
       "supported": "strong"
     },
     {
       "claim": "At request size 50, Fault Localization improvement over baseline is not statistically significant.",
       "evidence": "Table II reports p=0.056 for request size 50 with only a 3.85pp improvement (66.28% vs 62.44%). Only 7 complete repairs achieved by FL vs 0 by baseline.",
       "supported": "strong"
     },
     {
       "claim": "CodeLlama-7B-Instruct achieves the highest overall repair accuracy (92.3%) among four open-source LLMs.",
       "evidence": "Figure 5 and Table V show CodeLlama at 92.3% overall accuracy, with 54.3% complete repair rate at request size 30, compared to Llama (63.7%), DeepSeek-Coder (54.0%), and Granite (53.7%).",
       "supported": "strong"
     },
     {
       "claim": "CloudFix generates repairs that generalize to semantically similar unseen requests.",
       "evidence": "RQ4 results (Figure 6) show FL achieves 84.9% generalization accuracy on 15 held-out variant requests, compared to 82.8% repair accuracy on provided requests.",
       "supported": "moderate"
     },
     {
       "claim": "CloudFix is the first automated policy repair framework for cloud access control that combines formal methods with LLMs.",
       "evidence": "Section VII reviews related work and identifies no prior work combining LLMs with formal verification for access control policy repair. However, this is a novelty claim that depends on complete literature coverage.",
       "supported": "moderate"
     },
     {
       "claim": "CloudFix can repair policies using LLM-synthesized requests from natural language descriptions.",
       "evidence": "RQ5 (Figure 7) shows 86.4% FL accuracy and 85.2% baseline accuracy on 10 policies with GPT-5-generated requests. Very small sample (N=10) limits generalizability.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CloudFix combines SMT-based fault localization with LLM-driven repair to fix cloud access control policies, achieving 84% complete repair rate at 10-request complexity versus 48.2% for an unguided baseline. Performance degrades significantly at larger request sizes (50 requests), with only 7 complete repairs out of 282 policies. CodeLlama-7B-Instruct substantially outperforms three other open-source models of comparable size, and the approach shows ability to generalize to held-out semantically similar requests.",
   "red_flags": [
     {
       "flag": "No external baselines",
       "detail": "The only baseline is the authors' own simplified prompt approach. Prior symbolic repair tools (Eiers et al. 2023, D'Antoni et al. 2024, Xu and Peng 2016) are discussed in related work but not experimentally compared, making it impossible to assess whether CloudFix outperforms existing non-LLM approaches."
     },
     {
       "flag": "Contamination risk with public forum data",
       "detail": "The 282 policies were scraped from publicly available AWS re:Post forums. All evaluated LLMs could have been trained on this exact data. The policies and their correct versions may be in the models' training sets, which would inflate repair accuracy."
     },
     {
       "flag": "Synthetic request sets may not reflect real-world complexity",
       "detail": "Request sets are algorithmically generated (Alg. 4) rather than derived from real access logs. The paper acknowledges this in the discussion but the entire evaluation depends on these synthetic requests."
     },
     {
       "flag": "Very small sample for RQ5",
       "detail": "RQ5 uses only 10 policies (from the 45 with ground truth), a very small sample for drawing conclusions about natural language intent-driven repair. The 86.4% vs 85.2% difference between FL and baseline is likely not meaningful at this sample size."
     },
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature, top-p, max tokens, and other generation parameters are not reported despite being critical for LLM output quality. The paper states models used 'similar hyperparameters' without specifying values."
     },
     {
       "flag": "No variance across runs",
       "detail": "LLM outputs are stochastic but no repeated runs or variance measures are reported. All results appear to be single-run, making it impossible to assess result stability."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs for software engineering, directly relevant to assessing methodological quality of LLM-for-SE research."
     },
     {
       "title": "A systematic literature review on large language models for automated program repair",
       "authors": ["Q. Zhang", "C. Fang", "Y. Xie"],
       "year": 2024,
       "arxiv_id": "2405.01466",
       "relevance": "Survey of LLM-based automated program repair, the broader category that CloudFix's approach falls into."
     },
     {
       "title": "Hybrid automated program repair by combining large language models and program analysis",
       "authors": ["F. Li", "J. Jiang", "J. Sun", "H. Zhang"],
       "year": 2025,
       "relevance": "Closely related hybrid LLM+formal methods approach to automated program repair."
     },
     {
       "title": "A deep dive into large language models for automated bug localization and repair",
       "authors": ["S. B. Hossain", "N. Jiang", "Q. Zhou"],
       "year": 2024,
       "relevance": "Evaluates LLMs for bug localization and repair, related methodology for assessing LLM repair capabilities."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Early work on LLMs for automated program repair, benchmark evaluation methodology relevant to survey."
     },
     {
       "title": "Counterexample guided program repair using zero-shot learning and maxsat-based fault localization",
       "authors": ["P. Orvalho", "M. Janota", "V. M. Manquinho"],
       "year": 2025,
       "relevance": "Most closely related work: combines formal counterexample guidance with LLMs for program repair."
     },
     {
       "title": "Synthesizing access control policies using large language models",
       "authors": ["A. Vatsa", "P. Patel", "W. Eiers"],
       "year": 2025,
       "arxiv_id": "2503.11573",
       "relevance": "LLM-based access control policy synthesis from the same research group, directly related to policy generation evaluation."
     },
     {
       "title": "Empirical evaluation of generalizable automated program repair with large language models",
       "authors": ["V. Campos", "R. Shariffdeen", "A. Ulges", "Y. Noller"],
       "year": 2025,
       "arxiv_id": "2506.03283",
       "relevance": "Empirical evaluation of LLM-based program repair generalizability, related methodological assessment."
     },
     {
       "title": "A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions",
       "authors": ["L. Huang", "W. Yu", "W. Ma"],
       "year": 2025,
       "relevance": "LLM hallucination survey, relevant to understanding failure modes when LLMs generate incorrect policy repairs."
     }
   ]
 }