ai-research-survey

scan.json (25348B)

---

 {
   "paper": {
     "title": "LLM Agents for Generating Microservice-based Applications: How Complex is Your Specification?",
     "authors": ["Daniel M. Yellin"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.20119"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM agents using GPT-o3-mini perform well on medium-difficulty microservice specifications (83-94% unit test pass rate) but poorly on complex ones with authentication, external APIs, and intricate business logic (10-65%). Fine-grained code generation (per-request rather than per-service) significantly improves correctness. Reflection/regeneration helps weaker models (GPT-3.5) but has limited benefit for stronger models. Key error types include incorrect package usage, superficial API understanding, datatype conversion failures, and disregarding specification details.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repo URL: https://github.com/LLMs4code/LLMs4MSBA (footnote 2, Section 3.1) containing specifications and unit tests."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The specifications and unit tests are released, but the raw experimental results (generated code, error logs, per-run test outcomes) are not mentioned as released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile for the experimental harness, or detailed environment specification is provided in the paper. Docker is used for running generated code, but the full experimental environment is not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the agentic workflow is described (Figure 2) and the GitHub repo has specs and tests, no step-by-step instructions for reproducing the experiments are provided in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as averages (% passed) across experiments with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-o3-mini outperforms GPT-3.5 and that fine-grained generation improves results, but no statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as percentage of unit tests passed with clear baselines (e.g., 'average passing percent across all 4 request types for V0 and V1 code is 48.5% and 77.8% respectively... significantly better than 23.6% and 25.8% for coarse-grained generation')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "15-20 experiments per condition are used with no justification for why this number was chosen and no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only averages and highest scores are reported. No standard deviations, IQR, or other spread measures across the 15-20 runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three LLMs are compared (GPT-3.5, GPT-4o-mini, GPT-o3-mini) and 0-shot vs 1-shot prompts, plus coarse-grained vs fine-grained generation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "GPT-3.5 is dated. More importantly, no non-OpenAI models (Claude, Gemini, DeepSeek, open-source models) are included despite being relevant contemporary baselines for code generation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The fine-grained experiments (Section 6) serve as an ablation, generating code per-request-type to isolate the effect of granularity on code correctness."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: % unit tests passed (V0 and V1), highest number of tests passed, % V1 improved, fraction of testable code, and per-request-type breakdowns."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The authors manually examine error logs, generated code, and GPT-o3-mini reflection results (Section 5.7: 'we examined the run-time error-logs, the GPT-o3-mini reflection results, as well as the generated code')."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The MSBA specifications were custom-created (Section 3.1) specifically to avoid contamination. Unit tests were developed against ground truth implementations, not tuned to LLM outputs."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per microservice (Tables 1-6), per request type (Tables 8-9), per LLM, per prompt type (0-shot vs 1-shot), and per temperature (Table 10)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.7 provides detailed analysis of failure types: incorrect package usage, superficial API understanding, datatype conversion failures, and disregarding specification details, with specific code examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: 1-shot prompts degrade GPT-o3-mini performance, reflection hurts GPT-o3-mini V1 scores, providing explicit API details did not fix Dishes test failures due to other issues (Section 5.7)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about strong LLMs doing 'fairly well on medium difficulty' and 'poorly on higher difficulty' are supported by Tables 1-6. Fine-grained improvement claim is supported by Tables 8-9."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims fine-grained generation 'improves' correctness but the fine-grained prompts include additional information (authentication wrapper, import statements, Mongo collection name) acknowledged in Section 6, confounding the comparison."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract says 'agents using strong LLMs (like GPT-3o-mini)' generalizing from 2 applications with 8 microservices total, tested on only OpenAI models. The title frames this as about 'LLM Agents' broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, the poor restaurant performance could be due to prompt quality, test difficulty distribution, or other factors beyond specification complexity."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses unit test pass rate as a proxy for 'how well LLM Agents perform' without discussing limitations of this proxy. Not all unit tests are equal (acknowledged briefly in Section 5.3.2) but the gap between test pass rate and actual code correctness/quality is not addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to as 'GPT-3.5-16K', 'GPT-4o-mini', 'GPT-o3-mini' without specific version IDs or snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Appendix A (code generation, reflection, regeneration prompts in Figures 3-5), Appendix C (1-shot example in Figure 7), and Appendix D (fine-grained prompt in Figure 8)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings are reported: 0.0, 0.3, 0.5 for GPT-3.5 and GPT-4o-mini; default for GPT-o3-mini. MaxGen=2 is stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The LangGraph-based agentic workflow is described in detail in Section 5.1 with Figure 2 showing the full workflow: Init Agent → Code Agent → Docker Test Agent → Evaluate Agent → Regen Agent."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes package installation handling (omitting stdlib packages, substituting package names). Section 5.1 describes error message truncation for context window limitations."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion (Section 8) briefly restates findings but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Issues like small number of applications (2), limited model selection (only OpenAI), and custom-built benchmarks are not addressed as limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of limitations in generalizing from 2 MSBAs to the broader domain."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (generated code, error logs, per-run test outcomes) is not released. Only aggregate statistics are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes how MSBA specifications were created (iterative process with GT implementation and testing). Section 5.1 describes the experimental methodology."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM-generated code on custom benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from specification creation (Section 3.1) through code generation, testing in Docker containers, error collection, reflection, and regeneration (Section 5.1, Figure 2) is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation email (post.runi.ac.il, Reichman University) is provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training cutoff dates are stated for any of the models used (GPT-3.5, GPT-4o-mini, GPT-o3-mini)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1: 'First we want to make sure that the LLMs have not been contaminated - have not seen these MSes before.' Custom specifications were created specifically to avoid contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark was custom-created (Section 3.1) specifically to avoid contamination. The specifications were novel and not publicly available before the experiments."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time are reported despite running hundreds of LLM experiments across three models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget or API spend is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged across 15-20 runs but no seed sensitivity analysis is reported. Temperature variation is explored (Appendix E) but this is not the same as random seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Number of experiments is clearly stated: 15 for GPT-3.5 and GPT-o3-mini (Section 5.3), 15-20 for restaurant experiments (Section 5.4), 12-13 for fine-grained experiments (Tables 8-9)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Three temperature values are tested but no systematic hyperparameter search is described for prompts or other settings."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All configurations and their results are reported (all temperatures, all models, all prompt types) rather than cherry-picking a best configuration."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applicable. But many comparisons are made without any statistical framework."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors built both the specifications, ground truth implementations, and unit tests. No acknowledgment that this could bias the evaluation (e.g., tests might reflect the author's implementation assumptions)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-o3-mini is a reasoning model likely using significantly more compute than GPT-3.5, but compute costs are never compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether unit test pass rate on 2 custom MSBAs validly measures 'how well LLM Agents perform' on real-world microservice generation."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same LangGraph scaffold is used across all model comparisons (Section 5.1), so the scaffold is controlled."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Custom-created specifications (Section 3.1) eliminate temporal leakage since benchmarks did not exist before the experiments."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information not available in real usage. The 1-shot prompt and fine-grained prompts provide additional context that could be considered feature leakage in some framings."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 8 microservices come from only 2 applications, and results within an application are not independent (shared specifications, shared dependencies). This is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No formal leakage detection method is used. Custom benchmarks reduce contamination risk but no verification method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-o3-mini performs well on medium-difficulty MSBA specifications, averaging 83.5-94.4% unit test pass rate on the library application.",
       "evidence": "Table 3 (Section 5.3.1) shows V0 pass rates of 83.5-94.4% across library microservices.",
       "supported": "strong"
     },
     {
       "claim": "Performance degrades significantly on the more complex restaurant application, with GPT-o3-mini achieving only 10.8-65.8% V0 pass rates.",
       "evidence": "Table 6 (Section 5.5) shows V0 pass rates ranging from 10.8% (Ratings) to 65.8% (Authentication).",
       "supported": "strong"
     },
     {
       "claim": "Fine-grained code generation (per-request) improves correctness over coarse-grained (per-service) generation.",
       "evidence": "Tables 8-9 (Section 6): Dishes V0/V1 improve from 23.6%/25.8% to 48.5%/77.8%; Ratings from 10.8%/39.2% to 33.3%/63.2%.",
       "supported": "moderate"
     },
     {
       "claim": "Reflection/regeneration is effective for weaker models (GPT-3.5) but much less so for GPT-o3-mini.",
       "evidence": "Tables 1-2 show GPT-3.5 V1 improvement (e.g., 26.4%→50.2% for Cardholders). Tables 3-4 show GPT-o3-mini V1 often degrades (e.g., 94.4%→0.0% for Cardholders).",
       "supported": "moderate"
     },
     {
       "claim": "1-shot prompting improves GPT-3.5 performance but degrades GPT-o3-mini performance.",
       "evidence": "Comparing Tables 1 vs 2 (GPT-3.5 improves with 1-shot) and Tables 3 vs 4 (GPT-o3-mini mostly degrades with 1-shot).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small benchmark",
       "detail": "Only 2 applications with 8 total microservices. Conclusions about specification complexity and LLM performance are drawn from an extremely limited sample."
     },
     {
       "flag": "No statistical testing",
       "detail": "All comparisons are based on raw percentages with no significance tests, confidence intervals, or variance measures. With only 15-20 runs, differences could easily be due to chance."
     },
     {
       "flag": "Confounded fine-grained comparison",
       "detail": "The fine-grained experiments provide additional information (authentication wrapper, import statements, Mongo collection name) not in the coarse-grained prompts. The paper acknowledges this ('tempered by the knowledge that the fine-grained code is easier') but still claims improvement from the approach."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section despite significant scope limitations (2 apps, 3 OpenAI models, custom benchmarks, single author creating specs/GT/tests)."
     },
     {
       "flag": "Only OpenAI models tested",
       "detail": "Only OpenAI models (GPT-3.5, GPT-4o-mini, GPT-o3-mini) are evaluated. No open-source models, Claude, Gemini, or other providers are tested despite claims about 'LLM Agents' broadly."
     },
     {
       "flag": "Model naming error in abstract",
       "detail": "The abstract refers to 'GPT-3o-mini' (likely meaning GPT-o3-mini), and the conclusion also says 'GPT-3o-mini'. This inconsistency raises questions about care in manuscript preparation."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "relevance": "HumanEval benchmark paper, foundational for LLM code generation evaluation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM agents on real-world software engineering tasks."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin", "A. Odena", "M. I. Nye"],
       "year": 2021,
       "relevance": "MBPP benchmark for LLM code generation evaluation."
     },
     {
       "title": "What's wrong with your code generated by large language models? an extensive study",
       "authors": ["S. Dou", "H. Jia", "S. Wu"],
       "year": 2024,
       "relevance": "Extensive study of errors in LLM-generated code, directly relevant to understanding code generation quality."
     },
     {
       "title": "We have a package for you! A comprehensive analysis of package hallucinations by code generating LLMs",
       "authors": ["J. Spracklen", "R. Wijewickrama"],
       "year": 2024,
       "relevance": "Studies package hallucination in LLM code generation, a key error type found in this paper."
     },
     {
       "title": "Reflexion: language agents with verbal reinforcement learning",
       "authors": ["N. Shinn", "F. Cassano", "A. Gopinath", "K. Narasimhan", "S. Yao"],
       "year": 2023,
       "relevance": "Reflection technique used in this paper's agentic workflow for code regeneration."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["X. Wang", "B. Li", "Y. Song"],
       "year": 2025,
       "relevance": "Open platform for AI coding agents, relevant to agentic software engineering."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": ["C. S. Xia", "Y. Deng", "S. Dunn", "L. Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Alternative non-agentic approach to LLM-based software engineering."
     },
     {
       "title": "GitChameleon: Unmasking the version-switching capabilities of code generation models",
       "authors": ["N. Islah", "J. Gehring", "D. Misra"],
       "year": 2024,
       "arxiv_id": "2411.05830",
       "relevance": "Studies LLM capability to handle different package versions, relevant to dependency management challenges."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "relevance": "Reasoning model referenced for few-shot sensitivity observation relevant to prompt engineering."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["H. Yu", "B. Shen", "D. Ran"],
       "year": 2024,
       "relevance": "Defines runnable levels for code generation complexity, related to this paper's complexity metric."
     },
     {
       "title": "DI-BENCH: Benchmarking large language models on dependency inference with testable repositories at scale",
       "authors": ["L. Zhang", "J. Wang", "S. He"],
       "year": 2025,
       "relevance": "Benchmarks LLM dependency inference, directly related to the dependency challenges identified in this paper."
     }
   ]
 }