ai-research-survey

scan.json (19120B)

---

 {
   "paper": {
     "title": "Sensorium Arc: AI Agent System for Oceanic Data Exploration and Interactive Eco-Art",
     "authors": ["Noah Bissell", "Ethan Paley", "Joshua Harrison", "Juliano Calil", "Myungin Lee"],
     "year": 2025,
     "venue": "NeurIPS 2025 Creative AI Track",
     "arxiv_id": "2511.15997",
     "doi": "10.48550/arXiv.2511.15997"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper references the LLMUnity framework [7] and a video demo [10], but does not provide a repository URL for the Sensorium Arc system code itself."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The conversational corpus used for RAG is released on Zenodo (ref [23], doi: 10.5281/ZENODO.16777228). NASA EarthData datasets used are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists hardware (i9-13900HX, RTX 4090, 64GB RAM) and Unity version (6000.0.24f1) but provides no dependency specifications, requirements files, or library versions beyond naming the models used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology section describes the system architecture but not how to set it up or run it."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a system/art paper that does not report quantitative experimental results requiring statistical analysis."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative claims are made that would require significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative comparisons are reported."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No empirical study with sample sizes is conducted."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs with variance to report."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines or comparisons with other systems are provided. The paper only describes Sensorium Arc without comparing it to alternative approaches."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (Visualization Decider, RAG, Query Rewriter, Responder) but no ablation study examines their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No formal evaluation metrics are reported. The paper mentions 'average response latencies under 4 seconds' but provides no other quantitative evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is reported. The paper mentions exhibitions but provides no user study data or systematic human assessment."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No dataset-based evaluation is performed; the system is an interactive installation, not a benchmark evaluation."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative evaluation is performed that could be broken down by category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 (Discussion) describes limitations including RAG reliability issues, prompt interference in single-model designs, context window filling with too many retrieved paragraphs, and latency trade-offs with larger models."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that single-model designs 'suffered from issues such as prompt interference and opaque errors' and that retrieving more paragraphs (higher k) 'began to fill up the final LLM step's context window and muddle the response generation.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the system 'demonstrates the potential of conversational AI agents to mediate affective, intuitive access to high-dimensional environmental data' but no empirical evidence (user studies, metrics) supports this claim in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims earlier Sensorium iterations 'reveal[ed] that while sensory engagement was high, audiences sought direct dialogue with the ocean' and that 'this insight motivated' the AI narrator. No evidence is provided for this causal claim about audience preferences."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract proposes 'a new paradigm for human-machine-ecosystem' interaction based on a single installation with no formal evaluation. The conclusion claims a 'generalizable framework' without evidence of generalization."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for why participants engage with the system (novelty effect, exhibition context, etc.) or whether the AI component specifically drives engagement versus the immersive hardware."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the system enables 'affective, intuitive access' to data and can 'shift environmental data from static representation to participatory narrative' but does not measure these outcomes or acknowledge the gap between the system description and these claimed effects."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific models are named with sizes: Llama-3.2-3B, Gemma 3 12B, Qwen 8B (ref [8] specifies Qwen3-8B), all-MiniLM-L12-v2 for embeddings, Whisper Tiny for STT, Jets for TTS."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described in natural language (e.g., 'few-shot examples embedded in the system prompt', 'primed with general knowledge') but the actual prompt text is never provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only k=2 for retrieval and the 50cm proximity threshold are stated. No LLM hyperparameters (temperature, top-p, max tokens) are reported for any of the three agents."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent architecture is described in detail in Section 3 with a system diagram (Figure 2), including the Visualization Decider Agent, RAG/Query Rewriter Agent, and Responder Agent, with their communication flow and GBNF grammar constraints."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes preprocessing: 'unstructured resources were pre-processed into paragraphs, further split into sentences, embedded into a 384-dimensional vector space using the local embedding model all-MiniLM-L12-v2, and indexed in a lightweight local database.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 (Discussion) contains substantive discussion of limitations organized into four directions: RAG reliability, LLM agent performance, artistic advancement, and scientific/social impact."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific limitations are discussed: context window filling with retrieved paragraphs, GPU resource competition between LLM inference and Unity rendering, prompt interference in single-model designs, and the need for controlled user studies."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The Discussion identifies future work directions but does not bound the current claims to the tested setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data from system usage or exhibitions is released. The RAG corpus is on Zenodo but no interaction data or evaluation data is available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data sources are described: NASA EarthData for scientific datasets, Harrison archive materials for the RAG corpus, with specific documents named (The Time of the Force Majeure, Apologia Mediterraneo, Peninsula Europe I catalog)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study was conducted. Exhibition visitors are mentioned but not studied as participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw text to embedded vectors is documented: unstructured resources → paragraphs → sentences → 384-dim embeddings via all-MiniLM-L12-v2 → indexed in local database with ANN via usearch."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgments: Kuali Research Award (314284-00001), Arts for All, UMD Immersive Media Design, AlloSphere, Getty Foundation PST2024, Metabolic Studio of the Annenberg Foundation, and Joan and Irwin Jacobs Family Fund."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: University of Maryland College Park (3 authors), UC Santa Cruz Center for the Study of the Force Majeure, and Virtual Planet Technologies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are academic institutions and art foundations (UMD, Getty Foundation, Annenberg Foundation) with no apparent financial stake in the system's evaluation outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included. One author is affiliated with Virtual Planet Technologies, a company in the immersive media space, but no financial interests disclosure is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. The LLMs are used as components of an interactive system, not evaluated for their knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. The paper describes an art installation system without formal user studies."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Latency is reported: 'average response latencies under 4 seconds after user input' with CUDA acceleration on RTX 4090."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified: Intel Core i9-13900HX, 64GB RAM, NVIDIA RTX 4090 Laptop GPU, with CUDA acceleration and configurable GPU layer offloading. Unity version 6000.0.24f1 on Windows 11."
       }
     }
   },
   "claims": [
     {
       "claim": "Sensorium Arc demonstrates the potential of conversational AI agents to mediate affective, intuitive access to high-dimensional environmental data",
       "evidence": "System description in Section 3 and exhibition mentions in Section 4, but no empirical evaluation of user affect or intuition is provided.",
       "supported": "unsupported"
     },
     {
       "claim": "Earlier Sensorium iterations revealed that audiences sought direct dialogue with the ocean, motivating the AI narrator",
       "evidence": "Mentioned in Section 2 (Related Works) but no data from these prior exhibitions is presented.",
       "supported": "weak"
     },
     {
       "claim": "The modular multi-agent architecture improved control and debuggability over a single-model approach",
       "evidence": "Section 3 states single-model designs 'suffered from issues such as prompt interference and opaque errors' and cites multi-agent literature [31], but no comparative evaluation is provided.",
       "supported": "weak"
     },
     {
       "claim": "The system achieves average response latencies under 4 seconds",
       "evidence": "Stated in Section 3 (Hardware and Runtime Environment) with specific hardware configuration.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["case-study"],
   "key_findings": "Sensorium Arc is a multimodal interactive AI installation that personifies the ocean as a poetic speaker using a multi-agent LLM architecture with RAG grounding. The system uses Llama 3.2 3B, Gemma 3 12B, and Qwen 8B models in specialized roles (visualization selection, query rewriting, response generation) with a curated eco-art corpus. The paper is primarily a system description for an art installation presented at NeurIPS Creative AI Track, with no formal evaluation of its claimed effects on user engagement or understanding.",
   "red_flags": [
     {
       "flag": "No formal evaluation",
       "detail": "The paper makes claims about the system's potential to mediate 'affective, intuitive access' to environmental data and proposes a 'new paradigm,' but provides no user study, no metrics, and no empirical evidence. The Discussion acknowledges the need for 'controlled user studies on retention, empathy, and behavioral change' as future work."
     },
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The abstract claims a 'new paradigm for human-machine-ecosystem' interaction and the conclusion claims a 'generalizable framework,' but the evidence is a single art installation with no formal evaluation. These are aspirational claims presented as findings."
     }
   ],
   "cited_papers": [
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["M. Cemri", "M. Z. Pan", "S. Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "doi": "10.48550/arXiv.2503.13657",
       "relevance": "Directly addresses failure modes in multi-agent LLM systems, relevant to agentic AI reliability."
     },
     {
       "title": "Small LLMs Are Weak Tool Learners: A Multi-LLM Agent",
       "authors": ["W. Shen", "C. Li", "H. Chen"],
       "year": 2024,
       "doi": "10.48550/ARXIV.2401.07324",
       "relevance": "Multi-agent LLM architecture for tool use, relevant to agentic workflow design."
     },
     {
       "title": "Query Rewriting for Retrieval-Augmented Large Language Models",
       "authors": ["X. Ma", "Y. Gong", "P. He"],
       "year": 2023,
       "arxiv_id": "2305.14283",
       "doi": "10.48550/arXiv.2305.14283",
       "relevance": "RAG query rewriting technique used in the system, relevant to LLM retrieval methodology."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "doi": "10.48550/arXiv.2201.11903",
       "relevance": "Foundational prompting technique used in the system's query rewriter agent."
     },
     {
       "title": "A Survey on RAG with LLMs",
       "authors": ["M. Arslan", "H. Ghanem", "S. Munawar", "C. Cruz"],
       "year": 2024,
       "doi": "10.1016/j.procs.2024.09.178",
       "relevance": "Survey of retrieval-augmented generation techniques relevant to LLM capability evaluation."
     },
     {
       "title": "Reasoning LLMs for User-Aware Multimodal Conversational Agents",
       "authors": ["H. Rahimi", "J. Cattoni", "M. Beghili"],
       "year": 2025,
       "doi": "10.48550/ARXIV.2504.01700",
       "relevance": "User-aware multimodal LLM agent design, relevant to agentic AI interaction patterns."
     }
   ]
 }