ai-research-survey

scan.json (30861B)

---

 {
   "paper": {
     "title": "Deployability-Centric Infrastructure-as-Code Generation: Fail, Learn, Refine, and Succeed through LLM-Empowered DevOps Simulation",
     "authors": [
       "Tianyi Zhang",
       "Shidong Pan",
       "Zejun Zhang",
       "Zhenchang Xing",
       "Xiaoyu Sun"
     ],
     "year": 2025,
     "venue": "FSE",
     "arxiv_id": "2506.05623",
     "doi": "10.48550/arXiv.2506.05623"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "State-of-the-art LLMs achieve only 20.8–30.2% deployment success on first attempt for IaC generation, but the IaCGen iterative feedback framework raises this to 54.6–91.6% within 10 iterations across 6 models. Maintaining full conversation history reduces required iterations by 15.9% by preventing error recurrence. Despite high deployment rates, trustworthiness remains poor: only 25.2% of generated templates satisfy both resource and attribute user intent, and only 8.4% pass security compliance checks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Replication package released at https://github.com/Tianyi2/IaCGen (Section 9, Data Availability): 'The Code folder contains the code of our IaCGen framework.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package includes the benchmark: 'The Data folder contains the benchmarks' (Section 9, Data Availability)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states temperature=0 and 8000 max tokens, and mentions yamllint, cfn-lint, boto3, but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The replication package contains READMEs: 'Detailed descriptions of files can be found in the README.md file within each folder' (Section 9). The repository URL is provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 4, and 5 are reported as point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims of model differences (e.g., 'Claude models show impressive performance') are based on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: '≈24.7% passItr@1 to ≈74.8% passItr@15, a near 200% performance improvement' (Section 6.1), '15.9% reduction in required iterations' (Section 6.1), and percentage point gains for each feedback level."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the final benchmark size of 153 scenarios. The 51-instance subsample for user intent evaluation is described as 'randomly sampled' but no power analysis or sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments use temperature=0 for deterministic output and appear to be single-run. No variance, standard deviation, or spread measures are reported across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: passItr@1 (no iteration), IaCGen without conversation history (Fig. 7), general feedback only vs. detailed feedback (Fig. 6), and comparison with IaC-Eval benchmark (Section 6.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All six evaluated models are contemporary: GPT-4o, GPT-o3-mini, Claude-3.5, Claude-3.7, DeepSeek-R1, and DeepSeek-V3, representing state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Ablation on conversation history (Fig. 7, Section 6.1): IaCGen with vs. without full conversation history using Claude-3.5. Also, decomposition of feedback levels (no feedback / general / detailed) in Fig. 6."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics: passItr@n at various n (deployment success), user intent matching at resource and attribute levels (Table 4), three security compliance metrics (Table 5), and error stage distribution (Fig. 8)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation of system outputs is automated: deployment pass/fail via boto3, user intent matching via Checkov custom policies, security via Checkov. The human-in-the-loop component (RQ3) provides feedback during generation, not evaluation of outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 153 benchmark scenarios are used for evaluation with no explicit train/dev/test separation. The framework design (feedback structure, iteration limits) was developed with knowledge of the benchmark, though iteration limits are justified by prior work [22]."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns: by difficulty level (Fig. 4), by model (Tables 2, 4, 5), by error stage (Fig. 8), by error type (Table 3), and by feedback level (Fig. 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 6.2) provides detailed failure analysis: five primary error categories (Missing Value, Self-defined Property, Null Substitution, Unnecessary Whitespace, Arbitrary Default Value) with counts per model and explanations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results reported: poor initial pass rates (≈24.7%), poor user intent alignment (25.2% combined coverage), very low security compliance (8.4%), and GPT-4o's poor deployment error resolution."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims match results: 20.8–30.2% first-attempt rates (Table 2 passItr@1), 54.6–91.6% at 10 iterations (Table 2 passItr@10), >90% passItr@25 with human feedback (Fig. 9), 25.2% user requirement coverage (Table 4), 8.4% security compliance (Table 5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim (IaCGen's iterative feedback improves deployment success) is supported by controlled ablation: with vs. without conversation history (Fig. 7), and decomposition by feedback level (Fig. 6). These are adequate single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title ('Infrastructure-as-Code Generation') and abstract frame claims broadly, but the study is primarily CloudFormation-specific with only a small Terraform syntax-only extension. Section 7.4 acknowledges this but the title and abstract overclaim the generality."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 7.4 discusses threats (model evolution, benchmark scope, difficulty categorization) but these are generalizability concerns, not alternative explanations for the observed results. For example, whether similar improvement could come from simple retrying without structured feedback, or from prompt-only approaches, is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges that deployment success (the primary metric) does not capture full IaC quality, and supplements it with user intent matching (Section 6.4) and security compliance analysis. This distinction between deployment success and overall template utility is clearly stated."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified only by marketing names: 'GPT-4o, GPT-o3-mini, Claude-3.5, Claude-3.7, DeepSeek-R1, and DeepSeek-V3' (Section 5). No API version strings or snapshot dates. Section 7.4 notes 'we document the specific model and their cut-off date in our replication package' but the paper itself does not specify versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes prompt structure ('We need a CloudFormation template that creates [description]...') and states 'The complete prompts and other reproducible details mentioned in this section are available in our code repository' (Section 4.1) at https://github.com/Tianyi2/IaCGen."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5: 'All models are evaluated with consistent temperature settings of 0 to maximize deterministic outputs and configured with the 8,000 maximum output token limit.' Iteration limits (2 general + 4 detailed feedback) also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "IaCGen framework described in detail in Section 4 with workflow diagram (Fig. 5), three-stage validation pipeline, conversation history management, feedback mechanism design, and iteration logic."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the full preprocessing pipeline with counts at each stage: ~900 templates → 850 (size filtering) → 465 (syntax validation) → 200 (deployment testing) → 153 (rectification). Filtering criteria at each stage are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 'Threats to Validity' provides a substantive discussion of limitations across multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 identifies threats specific to this study: models available at time of writing may yield different results, 153 scenarios may not capture 'highly specialized configurations', difficulty categorization 'may not align perfectly with all organizational perspectives', and absence of standardized benchmarks across IaC languages."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 explicitly states boundaries: focus on CloudFormation, 153 scenarios across 58 services, specific model set, and acknowledged limitation to standardized benchmarks. Section 2.2 also explicitly explains the focus on CloudFormation over Terraform for methodological reasons."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at https://github.com/Tianyi2/IaCGen contains the benchmark data and framework code (Section 9, Data Availability)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes template sourcing from AWS documentation, AWS Samples GitHub, and a CloudFormation repository dataset [38], with ethical licensing checks and multi-stage preprocessing."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'one DevOps practitioner with three years of AWS and CloudFormation experience' and 'two additional DevOps practitioners with equivalent expertise' for benchmark construction, but does not describe how these practitioners were recruited or selected."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented in Section 3.1: ~900 initial templates → 850 after size filtering → 465 after syntax validation → 200 after deployment testing → 153 after rectification. Each stage's criteria are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, grants, or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Australian National University, NYU & Columbia University, Nanyang Technological University, and CSIRO's Data61. None are affiliated with the companies whose LLMs are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot be determined since no funding source is disclosed. Authors are at academic institutions not affiliated with evaluated LLM providers, suggesting likely independence, but no explicit disclosure."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No model training cutoff dates are stated in the paper. Section 7.4 mentions 'we document the specific model and their cut-off date in our replication package' but this information is not in the paper itself."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Templates are sourced from public AWS documentation and GitHub repositories, which are likely in LLM training data. No discussion of potential overlap between these public templates and model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark templates originate from public AWS CloudFormation sample templates [36] and GitHub repositories [37, 38], which are highly likely to appear in LLM training corpora. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The DevOps practitioners involved in benchmark construction and the cloud engineer providing feedback are part of the methodology design, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. This is a benchmark evaluation of LLMs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 reports total API cost ($230.75), per-model costs (Claude-3.7 at $0.42/template, DeepSeek-V3 at $0.04/template), and deployment cost ($0.04/template). Total minimum cost of $0.08 per deployable template is stated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states total LLM API cost ($230.75) and AWS deployment cost ($35.21), totaling $265.96 for the full evaluation. Per-template costs are broken down by model."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for deterministic output, but no multi-run analysis is performed to verify determinism across API calls or assess sensitivity to non-deterministic factors."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies single runs via temperature=0 but never explicitly states the number of experimental runs per configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The iteration limit allocation (2 general + 4 detailed) is justified by prior work [22] but no systematic search over framework parameters is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Framework configuration (feedback structure, iteration limits) is fixed based on prior work [22] but no systematic comparison of alternative configurations is provided to justify that the chosen design is optimal."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 6 models across multiple metrics and difficulty levels, making numerous implicit comparisons, but performs no statistical tests and therefore no multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors developed both the DPIaC-Eval benchmark and IaCGen framework, then evaluate the framework on their own benchmark. No acknowledgment of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Per-model API costs are reported (Section 5), but performance is not analyzed as a function of compute budget. Claude-3.7 ($0.42/template) vs DeepSeek-V3 ($0.04/template) represents a 10x cost difference but cost-performance tradeoff is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses construct validity by showing deployment success alone is insufficient: 42.7% of syntactically correct templates fail deployment (Section 3.3), and RQ4 demonstrates deployable templates still fail user intent (25.2%) and security (8.4%) tests, motivating multi-dimensional evaluation."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same IaCGen framework is used consistently across all 6 model comparisons, controlling for scaffold differences. Model comparisons are within the same scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark templates are sourced from public AWS documentation and GitHub repositories published before all evaluated models' training. No discussion of whether models may have seen these templates or similar ones during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the natural language prompts or error messages in the iterative loop provide information that constitutes leakage beyond what would be available in real-world usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple benchmark templates share AWS service families and structural patterns. No discussion of whether templates from the same service are structurally similar enough to inflate performance estimates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection methods (canary strings, membership inference, n-gram overlap analysis, decontamination) are applied despite the benchmark being sourced from publicly available templates."
       }
     }
   },
   "claims": [
     {
       "claim": "State-of-the-art LLMs achieve only 20.8–30.2% deployment success rate on first attempt for IaC generation",
       "evidence": "Table 2 passItr@1 results across 6 models on the 153-scenario DPIaC-Eval benchmark (Section 6.1)",
       "supported": "strong"
     },
     {
       "claim": "IaCGen achieves 54.6–91.6% deployment success within 10 iterations across all evaluated models",
       "evidence": "Table 2 passItr@10 results for all 6 models; Claude-3.5 achieves 91.6%, GPT-4o 54.6% (Section 6.1)",
       "supported": "strong"
     },
     {
       "claim": "Maintaining complete conversation history reduces required iterations by 15.9% compared to providing only current error",
       "evidence": "Ablation study on Claude-3.5 (Fig. 7, Section 6.1): 4.55 average iterations vs 5.41 baseline",
       "supported": "moderate"
     },
     {
       "claim": "Human-in-the-loop feedback helps all 6 models achieve over 90% passItr@25",
       "evidence": "Fig. 9 showing solid lines (with human feedback) exceeding 90% for all models at 25 iterations (Section 6.3)",
       "supported": "strong"
     },
     {
       "claim": "Only 25.2% of generated IaC templates fully satisfy both resource-level and attribute-level user intent",
       "evidence": "Table 4 Resource & Attribute column, averaged across all models on 51 sampled instances (Section 6.4)",
       "supported": "moderate"
     },
     {
       "claim": "Only 8.4% of deployable templates achieve full security compliance",
       "evidence": "Table 5 Filtered Compliance column, using Checkov security policy checks (Section 6.4)",
       "supported": "moderate"
     },
     {
       "claim": "IaCGen generalizes to Terraform, achieving 100% passItr@7 syntax accuracy",
       "evidence": "Section 6.1: Claude-3.5 on IaC-Eval Terraform benchmark achieves 79.7% passItr@1 and 100% passItr@7 with average 1.58 iterations",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests or uncertainty quantification",
       "detail": "All model comparisons are based on point estimates from single deterministic runs. No significance tests, confidence intervals, or variance measures are reported despite making explicit performance comparisons between models (e.g., 'Claude models show impressive performance')."
     },
     {
       "flag": "Benchmark contamination risk from public sources",
       "detail": "Templates are sourced from public AWS documentation and GitHub repositories that are highly likely in LLM training data. Models may have memorized similar or identical templates. This fundamental threat to validity is not discussed."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "Authors developed both the DPIaC-Eval benchmark and IaCGen framework, then evaluate IaCGen on their own benchmark without independent validation or acknowledgment of potential bias."
     },
     {
       "flag": "Small subsample for trustworthiness evaluation",
       "detail": "User intent matching (RQ4) is evaluated on only 51 randomly sampled instances from 153, without justification for the subsample size or analysis of whether this subsample is representative."
     },
     {
       "flag": "Conversation history ablation on single model only",
       "detail": "The key ablation study (with vs. without conversation history) is conducted only on Claude-3.5, the best-performing model. Results may not generalize to weaker models where conversation management could have different effects."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and pass@k metric for LLM code generation, used as comparison baseline."
     },
     {
       "title": "Iac-eval: A code generation benchmark for cloud infrastructure-as-code programs",
       "authors": ["Patrick T Kon", "Jiachen Liu", "Yiming Qiu"],
       "year": 2024,
       "relevance": "First IaC generation benchmark (Terraform), direct predecessor to DPIaC-Eval; found LLMs achieve <20% success."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Rigorous evaluation methodology for LLM-generated code quality and correctness."
     },
     {
       "title": "Using a feedback loop for llm-based infrastructure as code generation",
       "authors": ["Mayur Amarnath Palavalli", "Mark Santolucito"],
       "year": 2024,
       "arxiv_id": "2411.19043",
       "relevance": "Examined feedback loops for IaC generation, found effectiveness diminishes after 5 iterations; direct related work."
     },
     {
       "title": "Teaching large language models to self-debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli", "Denny Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Foundational work on using natural language feedback for iterative LLM code refinement."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Iterative self-refinement framework for LLMs, methodological ancestor to IaCGen's feedback approach."
     },
     {
       "title": "Deepseek-coder: When the large language model meets programming–the rise of code intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the evaluated model families (DeepSeek); demonstrates open-source LLM code generation capabilities."
     },
     {
       "title": "A survey of using large language models for generating infrastructure as code",
       "authors": ["Kalahasti Ganesh Srivatsa", "Sabyasachi Mukhopadhyay", "Ganesh Katrapati", "Manish Shrivastava"],
       "year": 2024,
       "arxiv_id": "2404.00227",
       "relevance": "Survey covering the landscape of LLM-based IaC generation approaches and challenges."
     },
     {
       "title": "Automated code generation for information technology tasks in yaml through large language models",
       "authors": ["Saurabh Pujar", "Luca Buratti", "Xiaojie Guo"],
       "year": 2023,
       "relevance": "Early work on LLM-based YAML/IaC code generation for IT automation tasks."
     },
     {
       "title": "Optimizing service deployments with nlp based infrastructure code generation-an automation framework",
       "authors": ["Hariharan Ragothaman", "Saai Krishnan Udayakumar"],
       "year": 2024,
       "relevance": "Attempted to address IaC deployability but with limited technical detail; direct related work to IaCGen."
     },
     {
       "title": "When ai takes the wheel: Security analysis of framework-constrained program generation",
       "authors": ["Yue Liu", "Zhenchang Xing", "Shidong Pan", "Chakkrit Tantithamthavorn"],
       "year": 2025,
       "arxiv_id": "2510.16823",
       "relevance": "Security analysis of LLM-generated code in constrained frameworks, related to the security compliance findings."
     },
     {
       "title": "A 3-codgen: A repository-level code generation framework for code reuse with local-aware, global-aware, and third-party-library-aware",
       "authors": ["Dianshu Liao", "Shidong Pan", "Xiaoyu Sun"],
       "year": 2024,
       "relevance": "Repository-level LLM code generation framework with context-aware approaches relevant to agentic code generation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "IaCGen addresses a real DevOps pain point and the replication package is released, but it's a research framework rather than a production-ready tool."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that iterative feedback helps is expected; the mildly surprising element is how poorly LLMs perform on first-attempt deployment (20–30%) despite strong general code generation abilities."
     },
     "fear_safety": {
       "score": 1,
       "justification": "The 8.4% security compliance rate raises concerns about deploying LLM-generated infrastructure, but this is an expected finding rather than a novel attack or existential concern."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or confrontational claims; the paper is a straightforward benchmark and framework evaluation."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and benchmark are released on GitHub, and a practitioner with AWS access could run the framework, though it requires AWS credentials and API keys."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (GPT-4o, Claude, DeepSeek) but the paper itself is from academic institutions without major brand recognition."
     }
   }
 }