ai-research-survey

scan.json (23719B)

---

 {
   "paper": {
     "title": "LLM Agent for Fire Dynamics Simulations",
     "authors": ["Leidong Xu", "Danyal Mohaddes", "Yi Wang"],
     "year": 2024,
     "venue": "Foundation Models for Science Workshop, NeurIPS 2024",
     "arxiv_id": "2412.17146",
     "doi": "10.48550/arXiv.2412.17146"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["case-study"],
   "key_findings": "FoamPilot, an LLM agent using GPT-4o with RAG, shell, and Python tools, achieves consistent success (5/5) on simple FireFOAM tasks (code insight, parameter modification) but success drops sharply with complexity: serial job execution 4/5, HPC job submission 1/5, multi-functionality queries 2/5. The agent's limited domain-specific knowledge of FireFOAM/OpenFOAM is identified as the primary bottleneck for complex tasks. RAG-based code navigation outperforms keyword search for semantic retrieval but remains vulnerable to ambiguous queries.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL for FoamPilot is provided. The paper mentions FireFOAM's GitHub repository (fireFoam-v1912) but does not release FoamPilot's source code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No test cases, evaluation data, or agent interaction logs are released. The tutorial cases used (poolFireMcCaffrey, burningBoxSuppression) are part of FireFOAM but the specific evaluation tasks and expected outputs are not published."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions LangChain/LangGraph 0.2, FAISS, GPT-4o, and text-embedding-ada-002, but provides no requirements.txt, Dockerfile, or detailed dependency listing with versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions. The paper describes the system architecture and prompts but not how to set up and run FoamPilot from scratch."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw success counts (e.g., 5/5, 4/5, 1/5) in Table 1 without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Differences between task success rates are presented as raw counts without any testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes reported. Results are only success/failure counts across 5 repetitions per task."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Each experiment was repeated 5 times with no justification for why 5 repetitions were chosen. No power analysis or discussion of whether 5 runs is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported. The paper notes 'the agent still exhibits some variability in success rates' but does not quantify this beyond success/failure counts."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines included. FoamPilot is not compared against any alternative approach (e.g., manual configuration, other LLM agents, MetaOpenFOAM). Only absolute success rates are reported."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines at all. The paper cites MetaOpenFOAM (Chen et al. 2024) as prior work but does not compare against it."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has three tools (Shell, Python, RAG) and three functionalities, but no ablation study is performed to measure the contribution of individual components. The multi-functionality test is not framed as an ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only a single metric is used: binary success/failure rate. No other metrics such as number of agent-tool loops, token usage, time to completion, or partial correctness are systematically reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states: 'a successful outcome is determined by comparing the agent's actions to that of an experienced FireFOAM user.' An expert manually assessed correctness of each run."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No separation between development and evaluation tasks. The prompts were iteratively developed (e.g., case configuration approach was changed after initial failure), and the same tasks appear to be used for both development and final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-task breakdown across all seven evaluation tasks, separated by functionality (Code Insight, Case Configuration, Job Execution, combined)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are discussed extensively throughout Section 3: the agent hallucinated file existence, made incorrect RAG queries, failed to set up the HPC environment correctly, and retrieved irrelevant information that polluted the context window."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "HPC job execution succeeded only 1/5 times. Multi-functionality queries succeeded 2/5. The paper also reports that 'on multi-functionality tests that were more complex than the relatively simple one presented here, the agent failed consistently.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Promising results were achieved for each functionality, particularly for simple tasks, and opportunities were identified for significant further improvement for more complex tasks.' This is appropriately hedged and supported by Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RAG 'offers a significant advantage over traditional user-driven keyword-based searches' (Section 2.1) and that the case configuration functionality 'makes FireFOAM more accessible' (Section 2.2), but no controlled comparison against keyword search or manual configuration is performed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'the challenges identified and the solution approaches developed are broadly applicable to other scientific simulation tools' (Section 1 and Section 4) without testing on any other simulation tool. Only FireFOAM was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered for the results. For example, the paper does not discuss whether the high success on simple tasks reflects GPT-4o's general capability vs. FoamPilot's specific design, or whether the failures stem from LLM limitations vs. prompt engineering issues."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures binary success/failure on specific tasks and frames results accordingly. It does not overclaim — it calls FoamPilot a 'proof-of-concept' and the measurements match the granularity of claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 specifies: 'Azure/OpenAI's GPT-4o, version 2024-05-13' for the LLM and 'text-embedding-ada-002' for embeddings. Specific version identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided: system prompt (Fig. 6), case configuration prompt (Fig. 4), and job execution prompts for both serial and HPC modes (Fig. 5). These are actual prompts, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states 'temperature setting of 0.0' and the embedding model's 'maximum input sizes' of 8192 tokens. Temperature is the most critical LLM hyperparameter and is explicitly reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agent architecture is described in detail in Section 2 with Figure 2: three-node graph (user, LLM, tools), static conditional edges, tool descriptions (Shell, Python Interpreter, RAG), RAG preprocessing pipeline (Fig. 3), termination conditions, and iterative loop structure."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents RAG preprocessing: combining .H and .C files, stripping boilerplate headers and licenses, prepending file paths, embedding with text-embedding-ada-002, and FAISS vector store creation. Section 2.2 describes case file compression into a single string."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 ('Conclusions and future work') contains substantive discussion of limitations: inability to handle complex tasks, lack of asynchronous execution, challenges with human feedback in LangChain, single LLM tested, limited domain knowledge."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: non-deterministic outputs even at temperature 0.0 (Section 3), ambiguous queries leading to incorrect RAG retrieval (Section 2.1), agent hallucinating file existence, and the risk that 'less experienced users would struggle to' detect incorrect but non-crashing configurations (Section 3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly states: 'Our experiments considered only GPT-4o for the agent's LLM,' lists unaddressed functionalities (asynchronous simulation, robust human feedback), and Section 3 notes testing was 'within a secure sandbox environment' on AWS."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No agent interaction logs, LLM outputs, or raw experimental data are released. Only summary success/failure counts in Table 1 are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the experimental setup: 'each experiment was repeated five times,' same GPT-4o version and temperature, dedicated AWS EC2 instance and parallelCluster, single user prompt per experiment."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The evaluation tasks are manually designed by the authors for a specific simulation code."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how evaluation tasks were selected, what criteria defined the correct outcomes, or why these specific tasks were chosen over others. The ground truth ('actions of an experienced FireFOAM user') is not formally documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding disclosure or acknowledgments section. The first author's footnote notes 'Work performed during internship at FM' but no funding sources are listed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Leidong Xu at University of Connecticut, Danyal Mohaddes and Yi Wang at FM, Research Division. The internship relationship is also disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two of three authors are from FM (FM Global), which develops FireFOAM. FM has a direct interest in FoamPilot succeeding, as it would enhance the usability of their own simulation tool. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests an agent system on practical simulation tasks, not a pre-trained model's capability on a benchmark. The LLM is used as a reasoning engine within a tool-augmented agent, not evaluated on benchmark knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the evaluation involves interactive agent tasks with a live codebase, not benchmark problems that could be in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no standardized benchmarks are used. The tasks are custom-designed for FireFOAM."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token usage, or wall-clock time per experiment are reported, despite using a commercial API (Azure/OpenAI GPT-4o) with potentially significant costs across 35+ experimental runs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated. The paper mentions using AWS EC2 and parallelCluster but provides no information on instance types, costs, or total compute used."
       }
     }
   },
   "claims": [
     {
       "claim": "FoamPilot's Code Insight functionality consistently retrieves and summarizes relevant source code, succeeding 5/5 times on both test tasks.",
       "evidence": "Table 1 shows 5/5 success rate for both solid energy equation and non-spherical drag equation tasks. Figure 7 shows a correct example output.",
       "supported": "moderate"
     },
     {
       "claim": "Case Configuration functionality consistently succeeds at modifying simulation parameters, achieving 5/5 on both test tasks.",
       "evidence": "Table 1 shows 5/5 for burner size modification and cardboard thickness modification. Figure 8 shows correct file modifications.",
       "supported": "moderate"
     },
     {
       "claim": "Job Execution success drops significantly for HPC environments (1/5) compared to serial execution (4/5).",
       "evidence": "Table 1 reports 4/5 for serial jobs and 1/5 for HPC jobs. Section 3 describes specific failure modes including incorrect SLURM commands and environment setup failures.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-functionality queries combining Code Insight and Case Configuration succeed 2/5 times, and more complex multi-functionality tasks fail consistently.",
       "evidence": "Table 1 shows 2/5 for the droplet drag model modification task. Section 3 states 'on multi-functionality tests that were more complex than the relatively simple one presented here, the agent failed consistently.'",
       "supported": "moderate"
     },
     {
       "claim": "RAG-based code navigation offers significant advantage over traditional keyword-based searches like grep and find.",
       "evidence": "Section 2.1 describes the conceptual advantages (semantic relationships, contextual relevance) but provides no empirical comparison against keyword search methods.",
       "supported": "weak"
     },
     {
       "claim": "The case configuration functionality makes FireFOAM more accessible to non-expert users.",
       "evidence": "Section 2.2 argues this conceptually, but no user study or comparison with manual configuration was performed. Only expert evaluation of correctness is provided.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small sample size",
       "detail": "Only 5 repetitions per task and only 2 tasks per functionality (7 total). With N=5, a single run difference changes the success rate by 20 percentage points. The paper acknowledges non-determinism but does not address whether 5 runs provides statistical power."
     },
     {
       "flag": "No baseline comparisons",
       "detail": "FoamPilot is not compared against any alternative (manual process, other LLM agents like MetaOpenFOAM, simpler automation scripts). Impossible to assess whether the LLM agent adds value over existing approaches."
     },
     {
       "flag": "Company evaluating enhancement to own product",
       "detail": "Two of three authors are from FM (FM Global), which develops FireFOAM. The paper evaluates FoamPilot's ability to enhance FireFOAM usability — a direct benefit to FM. This conflict is not acknowledged."
     },
     {
       "flag": "Possible cherry-picked tasks",
       "detail": "Only 'relatively simple' tasks are evaluated with full results. The paper notes complex tasks 'failed consistently' but these are not formally reported in Table 1, creating survivorship bias in the reported results."
     },
     {
       "flag": "No code or data release",
       "detail": "FoamPilot's implementation, evaluation tasks, expected outputs, and agent interaction logs are not released, preventing independent verification or reproduction."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM capability paper; GPT-4o used as the agent's LLM in this work."
     },
     {
       "title": "Code LLaMA: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code generation model relevant to LLM code generation capabilities."
     },
     {
       "title": "MetaGPT: Meta programming for multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Xiawu Zheng", "Jonathan Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent LLM framework for collaborative software development."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2024,
       "relevance": "Multi-agent system for software development using communicative LLM agents."
     },
     {
       "title": "RepairAgent: An autonomous, LLM-based agent for program repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Autonomous LLM agent for program repair, directly relevant to agentic code workflows."
     },
     {
       "title": "A survey on large language models for code generation",
       "authors": ["Juyong Jiang", "Fan Wang", "Jiasi Shen"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Survey of LLM code generation capabilities and limitations."
     },
     {
       "title": "Large language model-based agents for software engineering: A survey",
       "authors": ["Junwei Liu", "Kaixin Wang", "Yixuan Chen"],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Comprehensive survey of LLM-based agents for software engineering tasks."
     },
     {
       "title": "MetaOpenFOAM: an LLM-based multi-agent framework for CFD",
       "authors": ["Yuxuan Chen", "Xu Zhu", "Hua Zhou", "Zhuyin Ren"],
       "year": 2024,
       "arxiv_id": "2407.21320",
       "relevance": "Directly comparable system: LLM multi-agent framework for the same simulation toolbox (OpenFOAM)."
     },
     {
       "title": "A survey on large language model based autonomous agents",
       "authors": ["Lei Wang", "Chen Ma", "Xueyang Feng"],
       "year": 2024,
       "relevance": "Survey of autonomous LLM agent architectures and capabilities."
     },
     {
       "title": "LLM experiments with simulation: Large language model multi-agent system for process simulation parametrization in digital twins",
       "authors": ["Yuchen Xia", "Daniel Dittler", "Nasser Jazdi"],
       "year": 2024,
       "arxiv_id": "2405.18092",
       "relevance": "LLM agents for scientific simulation parametrization, directly relevant to agentic simulation workflows."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained model for code understanding, foundational for code-related NLP."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B Brown"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "GPT-3 paper establishing few-shot LLM capabilities foundational to agentic workflows."
     }
   ]
 }