ai-research-survey

scan.json (29100B)

---

 {
   "paper": {
     "title": "Multi-Agent Code-Orchestrated Generation for Reliable Infrastructure-as-Code",
     "authors": [
       "Rana Nameer Hussain Khan",
       "Dawood Wasif",
       "Jin-Hee Cho",
       "Ali Butt"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.03902",
     "doi": "10.48550/arXiv.2510.03902"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MACOG, a multi-agent orchestration with typed intermediate representation, constrained decoding, and validator-guided repair, consistently outperforms few-shot, chain-of-thought, multi-turn, and RAG baselines across 10 models on IaC-Eval. GPT-5 improves from 54.90 (RAG) to 74.02 (MACOG) and Gemini-2.5 Pro from 43.56 to 60.13. Ablations show the DevOps sandbox (-17.09), Security Prover (-12.57), and constrained decoding (-9.13) are the most critical components. Chain-of-thought provides no systematic gain over few-shot in this domain.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states 'All source code and evaluation scripts are archived on Zenodo at https://zenodo.org/records/17117489' and they reference the public IaC-Eval dataset at HuggingFace."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use the public IaC-Eval dataset (Section 7): 'We use the public IaC-Eval dataset, available at https://huggingface.co/datasets/autoiac-project/iac-eval.' Standard public benchmark used without modification."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or library version specifications anywhere in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions provided in the paper. The Zenodo archive is referenced but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-4 are point estimates only. The paper explicitly acknowledges this: 'we refrain from claiming statistical significance in the absence of per-item distributions in the tables' (Section 5.1). They mention performing internal bootstrapping but do not report CIs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 mentions 'in our internal runs we bootstrap task-level scores (1,000 resamples) to derive 95% confidence intervals and apply paired tests (randomization tests for BLEU/CodeBERTScore and Wilcoxon signed-rank for IaC-Eval)' but explicitly states they 'refrain from claiming statistical significance' in the paper. No test results are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute and relative improvements reported throughout: 'GPT-5 improves from 54.90 (RAG) to 74.02 (MACOG, +19.12)', 'relative improvement of roughly +35%', and per-ablation deltas in Section 5.2.3. Cohen's d mentioned as reported 'where appropriate' but actual values not shown."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why IaC-Eval's size is sufficient. No power analysis. The benchmark is adopted as-is without discussing whether the number of tasks provides adequate statistical power for the claimed comparisons."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measure reported in any table. All results are single point estimates. The paper acknowledges doing bootstrapping internally but does not report variance in the published results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five enhancement strategies compared: Few-shot, Chain-of-Thought, Multi-turn, RAG, and MACOG (Section 5.1). Each applied consistently across 10 models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models include GPT-5, GPT-4, Gemini-2.5 Pro, Gemini 2.0 Flash — contemporary frontier models. Enhancement strategies include RAG and multi-turn repair, which are current approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 presents a comprehensive ablation removing each of the 7 major MACOG components individually (Provider Harmonizer, Engineer, Reviewer, Security Prover, Cost & Capacity Planner, DevOps, Memory Curator) and measuring impact across all 4 metrics."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics used: BLEU, CodeBERTScore, LLM-judge, and IaC-Eval (Section 5.1). Equations 18-22 define each metric formally."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated IaC configurations. The LLM-judge metric uses a 'held-out judge' model for binary adequacy checks, but this is an automated LLM-based evaluation, not human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "While no fine-tuning is performed, the paper does not discuss whether prompt engineering, system instructions, or orchestration logic were developed independently of the IaC-Eval test tasks. No held-out validation split is mentioned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 mentions grouping 'tasks by coarse functional families (Networking, Compute, Storage, Identity and Access, Managed Services) and by approximate graph difficulty' but no per-category breakdown is shown in any table or figure. Only aggregate results reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.2 includes a 'qualitative audit of Gemini-2.5 Pro's baseline errors shows frequent security-group laxity and occasional missing encryption flags.' Section 5.2.1 discusses where strategies fail (CoT underperformance, RAG's schema drift issues)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "CoT consistently underperforming or matching Few-shot is explicitly discussed (Section 5.2.1): 'CoT does not provide systematic gains over Few-shot in this domain; for many models the two are statistically similar or CoT is slightly worse.' Ablation failures also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims ('GPT-5 improves from 54.90 (RAG) to 74.02', 'Gemini-2.5 Pro from 43.56 to 60.13', ablation drops to 64.89 and 56.93) are all directly supported by Tables 1 and 4. The claim 'MACOG is the top enhancement across models' is confirmed in Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('removing them drops IaC-Eval to 64.89 and 56.93') are supported by single-variable ablation in Table 4. The ablation design removes one component at a time while holding all else constant, which is an adequate causal identification strategy for component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Reliable Infrastructure-as-Code' broadly but evaluation is limited to Terraform on a single benchmark (IaC-Eval). No discussion of generalization to Pulumi, CloudFormation, Ansible, or other IaC tools mentioned in the introduction. The conclusion acknowledges 'broaden cross-provider coverage' as future work but the paper's claims are not bounded to Terraform."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. MACOG uses substantially more compute (multiple agent calls, validator executions, repair loops) than baselines. This compute confound is not discussed. No consideration of whether simpler interventions could achieve similar gains."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "IaC-Eval benchmark success is framed as 'deployable correctness' (Section 5.1) without discussing the gap between passing a benchmark harness and actual production deployment reliability. The paper does not acknowledge that IaC-Eval is a proxy for real-world infrastructure correctness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed as 'GPT-5', 'GPT-4', 'Gemini-2.5 Pro', 'GPT-3.5-turbo' etc. — marketing names without specific version identifiers, snapshot dates, or API version strings. No model version pinning mentioned."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.10 states 'The Architect and Engineer receive I-IR schemas and HCL grammars in system prompts and are steered by exemplars' but no actual prompt text is provided. Prompts are described functionally ('carefully engineered prompts, structured tool calling') but not reproduced."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states 'nucleus sampling p ∈ [0.7, 0.9] per model family, temperature ∈ [0.2, 0.5], and a maximum output budget sufficient to emit a self-contained Terraform module.' Ranges given per model family rather than exact values, but hyperparameters are explicitly reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Extensive scaffolding description across Sections 4.2-4.11: 8 agent roles described in detail, blackboard architecture, finite-state orchestrator (Eq. 14), I-IR typed representation, constrained decoding pipeline, counterexample-guided repair loop with Algorithm 1, and the Error-to-Edit mapping."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes using 'canonical task prompts provided by the benchmark' without modification. Task grouping by functional families described. The evaluation protocol is clearly documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. The conclusion mentions future work directions ('broaden cross-provider coverage', 'larger, more diverse corpora') but does not discuss limitations of the current evaluation or approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed anywhere in the paper. No consideration of specific limitations like single-benchmark evaluation, compute cost differences between strategies, or potential overfitting of the orchestration to IaC-Eval."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The conclusion mentions future extensions ('extend beyond IaC-Eval', 'broaden cross-provider coverage') which implicitly acknowledge current limitations, but there is no explicit statement of what the results do NOT show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate results reported in Tables 1-4. No per-task results available. The Zenodo archive contains 'source code and evaluation scripts' but the paper does not state that raw per-task outputs or scores are included."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "IaC-Eval benchmark clearly cited and described (Section 5.1): 'a benchmark comprising natural-language infrastructure intents and associated verification procedures tailored to cloud provisioning.' Tasks grouped by functional families."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Uses a standard benchmark dataset (IaC-Eval)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is described: benchmark inputs are canonical task prompts, outputs are complete Terraform programs, acceptance is determined by the IaC-Eval harness. Section 5.1 documents inference configuration, orchestration controls, and metric computation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section present in the paper. No mention of any funding source."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors listed with Virginia Tech affiliation. They evaluate commercial models (GPT-5, Gemini) but are not affiliated with those companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. Absence of funding disclosure prevents verification."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the 10 models evaluated. Models like GPT-5 and Gemini-2.5 Pro are used without specifying when their training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether IaC-Eval tasks or similar Terraform configurations appeared in the training data of any evaluated model."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "IaC-Eval was published in 2024 (NeurIPS). Models trained after 2024 (GPT-5, Gemini-2.5 Pro) may have seen its tasks. No contamination discussion provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Pure benchmark evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, latency, or per-task cost reported despite MACOG involving multiple LLM agent calls, validator executions, and repair loops per task."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, or total API spend reported. The compute cost difference between strategies (Few-shot uses one call; MACOG uses many) is never quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs producing the main results is never stated. Internal bootstrapping is mentioned but the primary run count is not specified."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No mention of how many configurations were tried for the orchestration parameters, prompt designs, or decoding settings. Temperature and top-p ranges are given but the search process is not documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the final configuration was selected. The paper presents results for a single MACOG configuration without discussing how it was chosen from possible alternatives."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "50 comparisons across 10 models × 5 strategies. No multiple comparison correction mentioned or applied. The paper explicitly states it refrains from claiming statistical significance."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement MACOG and all baseline strategies. No acknowledgment of the bias of evaluating their own system against their own implementations of competing approaches."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "MACOG uses multiple agent calls, constrained decoding, validator executions, and repair loops — substantially more compute than Few-shot or CoT. The paper claims 'each retry consumes identical compute budget for equity' only vs Multi-turn, not vs simpler baselines. No performance-per-compute analysis."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "IaC-Eval is adopted without any discussion of whether its tasks adequately represent real-world IaC challenges, whether its evaluation harness captures meaningful correctness, or how it compares to production IaC complexity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper holds the scaffold constant when comparing models (Table 1) and varies the scaffold systematically when comparing strategies. Section 5.1 states 'the same enhancement strategies and orchestration logic are applied to isolate the effect of the strategy rather than model-specific prompt engineering.'"
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. IaC-Eval was published in 2024; models trained after 2024 may have seen its tasks or solutions. Not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (e.g., RAG retrieval of similar tasks, prompt examples) leaks information that wouldn't be available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether IaC-Eval tasks share structural similarities that could inflate performance estimates or whether independence between tasks holds."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied (no canary strings, membership inference, temporal splits, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "MACOG is the top enhancement across models, with GPT-5 improving from 54.90 (RAG) to 74.02 and Gemini-2.5 Pro from 43.56 to 60.13 on IaC-Eval.",
       "evidence": "Table 1 (Section 5.2.1) shows consistent MACOG > RAG > Multi-turn > CoT ≈ Few-shot ordering across all 10 models. Average uplift of MACOG over RAG is +7.3 absolute points (~35% relative).",
       "supported": "moderate"
     },
     {
       "claim": "Ablations show constrained decoding and deploy feedback are critical: removing them drops IaC-Eval to 64.89 and 56.93 respectively.",
       "evidence": "Table 4 (Section 5.2.3) presents single-variable ablation on GPT-5. DevOps sandbox removal causes largest drop (-17.09), followed by Security Prover (-12.57) and constrained decoding (-9.13).",
       "supported": "moderate"
     },
     {
       "claim": "Chain-of-thought does not provide systematic gains over few-shot in the IaC domain.",
       "evidence": "Table 1 shows CoT ≈ Few-shot or slightly worse for most models. Tables 2-3 show CoT underperforming Few-shot on IaC-Eval for both GPT-5 (10.19 vs 12.53) and Gemini-2.5 Pro (10.49 vs 12.18).",
       "supported": "moderate"
     },
     {
       "claim": "MACOG produces proof-carrying bundles for offline verification including policy proofs, cost sheets, deploy logs, and compiler provenance.",
       "evidence": "Section 4.9 describes the evidence bundle conceptually but no empirical validation of bundle completeness or usefulness is provided.",
       "supported": "weak"
     },
     {
       "claim": "The MACOG ranking largely mirrors the base-model ranking, implying the orchestration is complementary to raw model capability rather than a substitute.",
       "evidence": "Table 1 shows that the model ordering (GPT-5 > Gemini-2.5 Pro > GPT-4 > ...) is preserved across strategies including MACOG (Section 5.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "MACOG improves LLM-judge from 69.72 to 94.10 for GPT-5, indicating higher alignment with intent and best practices.",
       "evidence": "Table 2 (Section 5.2.2). The large LLM-judge jump is attributed to the reviewer's interface checks and security prover's targeted patches.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results in Tables 1-4 are point estimates. The authors acknowledge performing internal bootstrapping (1,000 resamples) but explicitly 'refrain from claiming statistical significance' and do not report any uncertainty measures. This makes it impossible to assess whether observed differences are meaningful."
     },
     {
       "flag": "Compute cost confound not addressed",
       "detail": "MACOG uses multiple LLM agent calls, constrained decoding, validator executions, and iterative repair loops per task — likely orders of magnitude more compute than Few-shot or CoT. The paper never quantifies this cost difference. Improvements may simply reflect more compute spent, not architectural innovation."
     },
     {
       "flag": "LLM-judge metric without validation",
       "detail": "The LLM-judge metric uses an unnamed LLM to assess output adequacy. No inter-rater agreement with human judges is reported. No analysis of judge model bias. The 94.10% LLM-judge score for MACOG-GPT-5 is remarkably high and may reflect LLM-judge tendency to prefer structured, verbose outputs."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or scope boundaries section. This is a significant omission for a systems paper making broad claims about 'reliable Infrastructure-as-Code.'"
     },
     {
       "flag": "Monotonically improving results across strategies",
       "detail": "Table 1 shows perfectly monotonic improvement from Few-shot → MACOG for every single model (10/10). This clean ordering across such diverse model families and capability levels is unusual and warrants scrutiny."
     },
     {
       "flag": "Per-category breakdown promised but not delivered",
       "detail": "Section 5.1 describes grouping tasks by functional families (Networking, Compute, Storage, etc.) and graph difficulty, but no per-category results are ever shown. Aggregate results could mask important variation."
     }
   ],
   "cited_papers": [
     {
       "title": "Iac-eval: A code generation benchmark for cloud infrastructure-as-code programs",
       "authors": ["Patrick T Kon", "Jiachen Liu", "Yiming Qiu", "Weijun Fan", "Ting He"],
       "year": 2024,
       "relevance": "Primary benchmark used for evaluation; demonstrates that LLMs solve only 19% of IaC tasks on first try."
     },
     {
       "title": "Chatdev: Communicative agents for software development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2023,
       "arxiv_id": "2307.07924",
       "relevance": "Foundational multi-agent code generation framework with specialized roles that MACOG builds upon."
     },
     {
       "title": "Self-collaboration code generation via chatgpt",
       "authors": ["Yihong Dong", "Xue Jiang", "Zhi Jin", "Ge Li"],
       "year": 2024,
       "relevance": "Demonstrates single-model multi-role code generation through iterative self-collaboration."
     },
     {
       "title": "Competition-level code generation with alphacode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Demonstrates coupling code generation with validation/filtering for competitive programming."
     },
     {
       "title": "Repairagent: An autonomous, llm-based agent for program repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Autonomous LLM-based program repair agent with finite-state tool controller."
     },
     {
       "title": "Magis: Llm-based multi-agent framework for github issue resolution",
       "authors": ["Wei Tao", "Yucheng Zhou", "Yanlin Wang"],
       "year": 2024,
       "relevance": "Multi-agent framework for automated GitHub issue resolution with coordinated planning and QA."
     },
     {
       "title": "SynCode: LLM generation with grammar augmentation",
       "authors": ["Shubham Ugare", "Tarun Suresh", "Hangoo Kang"],
       "year": 2024,
       "relevance": "Grammar-constrained LLM decoding using precomputed DFA-based masks for syntactic validity."
     },
     {
       "title": "PICARD: Parsing incrementally for constrained auto-regressive decoding from language models",
       "authors": ["Torsten Scholak", "Nathan Schucher", "Dzmitry Bahdanau"],
       "year": 2021,
       "arxiv_id": "2109.05093",
       "relevance": "Constrained decoding via incremental parsing for structured code generation."
     },
     {
       "title": "Grammar-aligned decoding",
       "authors": ["Kanghee Park", "Jiayu Wang", "Taylor Berg-Kirkpatrick"],
       "year": 2024,
       "relevance": "Addresses distributional bias from hard grammar constraints in LLM decoding."
     },
     {
       "title": "Code repair with llms gives an exploration-exploitation tradeoff",
       "authors": ["Hao Tang", "Keya Hu", "Jin Zhou"],
       "year": 2024,
       "relevance": "Formalizes LLM code repair as exploration-exploitation, relevant to MACOG's repair loop."
     },
     {
       "title": "A Survey on Code Generation with LLM-based Agents",
       "authors": ["Yihong Dong", "Xue Jiang", "Jiaru Qian"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Recent survey on LLM-based agent code generation approaches."
     }
   ]
 }