ai-research-survey

scan.json (25157B)

---

 {
   "paper": {
     "title": "CloudEval-YAML: A Practical Benchmark for Cloud Configuration Generation",
     "authors": [
       "Yifei Xu",
       "Yuning Chen",
       "Xumiao Zhang",
       "Xianshang Lin",
       "Pan Hu",
       "Yunfei Ma",
       "Songwu Lu",
       "Wan Du",
       "Z. Morley Mao",
       "Ennan Zhai",
       "Dennis Cai"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2401.06786"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'We release the dataset along with the evaluation framework at https://github.com/alibaba/CloudEval-YAML.' A GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset of 1011 problems is released as part of the GitHub repository mentioned in the abstract: https://github.com/alibaba/CloudEval-YAML."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the evaluation cluster hardware (64 4-core 8GB machines, Minikube, Docker) but does not provide a requirements.txt, Dockerfile, or detailed environment setup with library versions for reproducing the benchmark framework."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the code is released, the paper itself does not contain step-by-step reproduction instructions. The evaluation platform architecture is described at a high level (§3), but specific commands or a reproducibility guide are not provided in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4, 5, and 6 are reported as single point estimates without confidence intervals or error bars. No uncertainty quantification is provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes multiple comparative claims (e.g., 'Proprietary models outperform open-source models by a large gap') but uses no statistical significance tests. All comparisons are based on raw score differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative differences with baselines, e.g., 'GPT-3.5 and GPT-4 score 0.412 and 0.515, which is 4.84× and 6.06× as that of llama-2-70b-chat' (§4.1), and '20-sample generation could improve the unit test score of Llama-2-70B/PaLM-2/GPT-3.5 by 30%/37%/39%' (§4.2). These provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset has 1011 problems (337 original + augmented), but there is no justification for why this size is sufficient, no power analysis, and no discussion of whether the sample provides adequate statistical power for the comparisons made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be single-run evaluations. The multi-sample generation experiment (§4.2) reports pass@k but without variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 12 LLMs against each other (Table 4), including both open-source and proprietary models of varying sizes, providing comparative baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include GPT-4, GPT-3.5, PaLM-2, Llama-2, Code Llama, and WizardCoder, which were all contemporary state-of-the-art models at the time of submission (late 2023)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates multiple dimensions: data augmentation effects (Table 5 comparing original/simplified/translated), multi-sample generation (§4.2), few-shot prompting (§4.3), and the unit test prediction classifier (§4.4). These serve as ablation-like analyses of different factors affecting performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six metrics are used across three categories: text-level (BLEU, Edit Distance, Exact Match), YAML-aware (Key-Value Exact Match, Key-Value Wildcard Match), and function-level (Unit Test). Described in §3.2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "There is no human evaluation of the LLM-generated YAML outputs. All evaluation is automated through the six metrics. Given that the paper claims practical relevance for cloud developers, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper that evaluates models on the full dataset. There is no model training or tuning on the CloudEval-YAML data that would require a held-out split (except the XGBoost classifier in §4.4, which uses leave-one-model-out cross-validation)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 provides per-category breakdowns across four perspectives: application category (Kubernetes/Envoy/Istio), code context presence, answer length, and question token count. Table 5 breaks down by dataset type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 includes a detailed failure mode analysis (Figure 7) that categorizes failures into 6 modes from empty responses to valid YAML that fails unit tests, with per-model breakdowns for GPT-4, Llama-2-70B, and Llama-2-7B."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: few-shot prompting 'does not yield significant improvements' (§4.3), code-specific models 'typically perform poorly on CloudEval-YAML compared to general LLMs' (§4.1), and the unit test predictor has high relative error up to 80% (§4.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: the dataset has 1011 problems (Table 2), the evaluation platform achieves 20x speedup (Figure 5), and the paper evaluates 12 LLMs (Table 4). The claims are descriptive and accurately reflected in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims, e.g., 'Simplification of problems generally leads to lower performance' (§4.1) and attributes code LLM poor performance to 'the dataset used in the fine-tuning process' (§4.1). These are observational comparisons presented with causal language but without controlled experiments isolating these factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Cloud Configuration Generation' broadly, but the benchmark covers only Kubernetes, Envoy, and Istio YAML configurations. Many cloud configuration formats (Terraform, Ansible, CloudFormation, Pulumi, Docker Compose) are not covered. The paper does not explicitly bound its claims to these three tools."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the large gap between proprietary and open-source models could be due to training data differences, model size, or RLHF, but these are not explored. The hypothesis about code LLMs' poor performance is stated as speculative ('It may be related to...') without analysis."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to as 'GPT-3.5', 'GPT-4', 'PaLM-2-bison' without specific version strings or snapshot dates. For open-source models, sizes are given (e.g., 'Llama-2-70b-chat', 'Wizardcoder-34b-v1.0') but for proprietary models, no API version or snapshot date is specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full prompt template is provided in Appendix B: 'You are an expert engineer in cloud native development. According to the question, please provide only complete formatted YAML code as output without any description...' The few-shot examples are in Appendix C."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "For the multi-sample generation experiment, Llama-2-70B hyperparameters are stated (temperature=0.75, top_p=0.9, top_k=50), but for proprietary models, the paper says 'We leave parameters that control the randomness of the output to default.' The default values for the main benchmark (Table 4) are not specified for any model."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation uses single-turn prompt-response generation from LLMs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the post-processing pipeline for extracting YAML from LLM responses, including specific rules for removing content before keywords, extracting delimited blocks. Section 2.2 documents the data augmentation process with GPT-4 + manual review."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion (§6) mentions future work directions but does not discuss limitations of the current approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues such as the limited scope of cloud tools covered, the representativeness of the problem selection, or the reliability of unit tests as the sole functional correctness measure."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its findings to Kubernetes/Envoy/Istio, does not discuss what types of cloud configurations are excluded, and does not note limitations of unit-test-only evaluation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset and evaluation framework are released at https://github.com/alibaba/CloudEval-YAML, which would include the raw problems, reference YAML files, and unit test scripts."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the data collection process: problems were hand-picked from official documentation websites, popular StackOverflow issues, and highly-ranked blog posts, with explicit guidelines for selection (clearly defined, diverse, avoiding ambiguity)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for the evaluation. The data augmentation involved a survey with a cloud provider operations team, but this is dataset construction, not a human subjects study. The benchmark is a standard dataset evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from problem selection (§2.1) through data augmentation (§2.2) to evaluation (§3) is documented, including the statistics at each stage (337 original problems, augmented to 1011). Table 1 shows statistics of the augmentation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. Given that 6 of 11 authors are affiliated with Alibaba Cloud, funding disclosure would be expected."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: '1 Alibaba Cloud 2 University of Michigan 3 UCLA 4 UC Merced'. The Alibaba Cloud affiliation is prominently stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Six authors are from Alibaba Cloud, a major cloud provider. The benchmark evaluates LLM capabilities for cloud configuration generation, which is directly relevant to Alibaba's business interests. This creates a potential conflict of interest that is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper. Alibaba Cloud authors may have financial interests in demonstrating the need for better cloud configuration tools."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 12 models evaluated. This is relevant because some problems are derived from StackOverflow and documentation that may be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether problems sourced from official documentation or StackOverflow could have been seen by the models during training. This is a significant concern given that the problems are drawn from public online sources."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Problems are sourced from 'official documentation websites, popular issues from StackOverflow, and highly-ranked blog posts' — all public sources likely in LLM training data. The paper does not discuss this contamination risk at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The paper is a benchmark evaluation of LLMs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The paper is a benchmark evaluation of LLMs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The paper is a benchmark evaluation of LLMs."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The paper is a benchmark evaluation of LLMs."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The paper is a benchmark evaluation of LLMs."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The paper is a benchmark evaluation of LLMs."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The paper is a benchmark evaluation of LLMs."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides a detailed cost breakdown: GPT-3.5 inference costs $0.60, Llama-7b costs $2.90. Section 3.4 discusses per-1k-token costs. Section 4.2 discusses cost tradeoffs between multi-sample GPT-3.5 vs. single-sample GPT-4."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports cloud evaluation costs ($0.71-$5.51 depending on instance type). The evaluation cluster specs are described: 64 4-core 8GB machines. Total benchmark running cost ranges from $1.31 to $8.41 per run."
       }
     }
   },
   "claims": [
     {
       "claim": "Proprietary models (GPT-3.5, GPT-4) outperform open-source models by a much larger gap on CloudEval-YAML than on HumanEval.",
       "evidence": "Table 4 shows GPT-4 unit test score of 0.515 vs Llama-2-70b-chat at 0.085 (6.06x gap). On HumanEval the gap is only 2.24x (§4.1).",
       "supported": "strong"
     },
     {
       "claim": "Dedicated code generation models perform poorly on cloud configuration compared to general LLMs of similar or smaller size.",
       "evidence": "Table 4: wizardcoder-34b-v1.0 scores 0.056 unit test, while llama-2-13b-chat scores 0.067 with less than half the model size (§4.1).",
       "supported": "moderate"
     },
     {
       "claim": "Multi-sample generation can improve unit test scores by 30-39% and make cheaper models cost-competitive with expensive ones.",
       "evidence": "Figure 8 and §4.2: 20-sample generation improves Llama-2-70B/PaLM-2/GPT-3.5 by 30%/37%/39%. GPT-3.5 with 6 samples beats GPT-4 with 1 sample at 30x lower cost.",
       "supported": "moderate"
     },
     {
       "claim": "Few-shot prompting does not yield significant improvements in cloud configuration generation.",
       "evidence": "Table 6: GPT-3.5 improves from 142 to 154 (+12) with 3 shots, but Llama-2-70b-chat decreases from 30 to 29 (-1). Results are inconsistent across models (§4.3).",
       "supported": "moderate"
     },
     {
       "claim": "The scalable evaluation platform achieves 20x speedup over single-machine evaluation.",
       "evidence": "Figure 5 shows 10.3 hours on single machine vs ~0.5 hours with 64 workers and caching, a 20x improvement (§3.3).",
       "supported": "strong"
     },
     {
       "claim": "CloudEval-YAML is the first hand-written dataset targeting cloud-native applications.",
       "evidence": "Table 7 compares against 15 other benchmarks. None target cloud-native YAML configuration. The closest is Ansible-YAML but it is scraped from GitHub/GitLab, not hand-written (§5).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CloudEval-YAML is a benchmark of 1011 hand-written YAML configuration generation problems for cloud-native applications (Kubernetes, Envoy, Istio), with unit tests for functional correctness evaluation. An evaluation of 12 LLMs shows proprietary models (GPT-4: 51.5% pass rate) dramatically outperform open-source models (best open-source Llama-2-70b: 8.5%), with a gap much larger than seen on general-purpose code benchmarks like HumanEval. Multi-sample generation is found to be an effective strategy, with GPT-3.5 at 6 samples matching GPT-4's single-sample performance at 1/30th the cost. Few-shot prompting showed no significant benefit for this domain.",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "Problems are sourced from official documentation, StackOverflow, and blog posts — all likely in LLM training data. The paper does not discuss whether models may have seen these exact problems during training, which could inflate scores for models trained on more web data."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. For a benchmark paper, this is concerning — issues like benchmark scope, unit test reliability, and problem representativeness should be discussed."
     },
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "All model comparisons are based on single-run point estimates without confidence intervals, significance tests, or variance reporting. Claims like 'proprietary models outperform open-source by a large gap' lack statistical support."
     },
     {
       "flag": "Potential conflict of interest",
       "detail": "6 of 11 authors are from Alibaba Cloud, a major cloud provider with business interest in cloud configuration tooling. No conflicts of interest are disclosed, and no funding statement is provided."
     },
     {
       "flag": "Overly broad title relative to scope",
       "detail": "The title claims 'Cloud Configuration Generation' but the benchmark covers only Kubernetes, Envoy, and Istio YAML. Many important cloud configuration formats (Terraform, CloudFormation, Ansible, Pulumi, Docker Compose) are not included."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the most widely used LLM code generation benchmark, which CloudEval-YAML directly compares against."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for Python code generation, another key comparison point for code LLM evaluation."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Describes Llama 2, one of the primary open-source LLMs evaluated in the benchmark."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Describes Code Llama, a code-specialized LLM that surprisingly underperforms general LLMs on cloud configuration tasks."
     },
     {
       "title": "WizardCoder: Empowering code large language models with evol-instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Describes WizardCoder, another code-specialized LLM evaluated in the benchmark that underperforms general LLMs."
     },
     {
       "title": "Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": ["Priyan Vaithilingam", "Tianyi Zhang", "Elena L. Glassman"],
       "year": 2022,
       "relevance": "Studies the usability and productivity impact of LLM-based code generation tools, relevant to the practical motivation of the benchmark."
     },
     {
       "title": "PaLM 2 technical report",
       "authors": ["Rohan Anil", "Andrew M. Dai", "Orhan Firat"],
       "year": 2023,
       "arxiv_id": "2305.10403",
       "relevance": "Describes PaLM 2, one of the proprietary LLMs evaluated in the benchmark."
     },
     {
       "title": "DS-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Yuhang Lai", "Chengxi Li", "Yiming Wang"],
       "year": 2023,
       "relevance": "A domain-specific code generation benchmark (data science) with unit tests, sharing design philosophy with CloudEval-YAML."
     },
     {
       "title": "Automated code generation for information technology tasks in YAML through large language models",
       "authors": ["Saurabh Pujar", "Luca Buratti", "Xiaojun Guo"],
       "year": 2023,
       "arxiv_id": "2305.02783",
       "relevance": "Focuses on Ansible YAML code generation with LLMs, the most closely related prior work to CloudEval-YAML."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Large-scale code generation benchmark with unit tests, relevant to the evaluation methodology of LLM coding capability."
     }
   ]
 }