ai-research-survey

scan.json (34512B)

---

 {
   "paper": {
     "title": "Transforming Wearable Data into Personal Health Insights using Large Language Model Agents",
     "authors": [
       "Mike A. Merrill",
       "Akshay Paruchuri",
       "Naghmeh Rezaei",
       "Geza Kovacs",
       "Javier Perez",
       "Yun Liu",
       "Erik Schenck",
       "Nova Hammerquist",
       "Jake Sunshine",
       "Shyam Tailor",
       "Kumar Ayush",
       "Hao-Wei Su",
       "Qian He",
       "Cory Y. McLean",
       "Mark Malhotra",
       "Shwetak Patel",
       "Jiening Zhan",
       "Tim Althoff",
       "Daniel McDuff",
       "Xin Liu"
     ],
     "year": 2024,
     "venue": "Nature Communications",
     "arxiv_id": "2406.06464",
     "doi": "10.1038/s41467-025-67922-y"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 states 'Our data and code are available at this link' (present tense, hyperlink lost in text extraction). Published in Nature Communications which requires data/code availability."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it releases synthetic wearable data and the evaluation dataset of 4000+ questions: 'Release a set of high-fidelity synthetic wearable data' and 'Release a personal health insights evaluation dataset' (Section 1)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment setup is described. The paper mentions using Python with Pandas in a 'customized sandbox runtime environment' but provides no dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The few-shot examples and architecture are described, but no README-style instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 caption states '95% bootstrapped confidence intervals are shown as error bars.' Figure 6 also shows '95% bootstrapped confidence intervals.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Wilcoxon signed-rank tests are used throughout: '(*) designates p < 0.05 using the Wilcoxon signed-rank test' (Figure 3 caption)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute performance levels with baselines: 'PHIA achieves an exact match accuracy of 84%, significantly outperforming the Code Generation baseline (74% accuracy), Numerical Reasoning (22% accuracy)' (Section 4.3). Also: 'surpassing two commonly used baselines by 282% and 14% respectively' (Section 5)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the sample sizes: 4000 objective queries, 172 open-ended queries, 56 synthetic users (only 4 used for evaluation), 12 reasoning annotators, 7 code quality experts. No power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "95% bootstrapped confidence intervals are reported for the main results (Figures 3, 6), which convey variance/spread information across the evaluated samples."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are compared: Numerical Reasoning, Code Generation, PH-LLM (Cosentino et al. 2024), and a custom chain-of-thought GPT-4 approach (Englhardt et al. 2024). Described in Section 4.1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "PH-LLM (2024) and GPT-4 chain-of-thought (2024) are contemporary. The Code Generation baseline uses the same Gemini 1.0 Ultra model. These represent the state of the art for this task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure A.1 shows 'PHIA - No Search' ablation, removing the web search component to isolate its contribution to domain knowledge and personalization ratings."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: exact match accuracy for objective queries, 8 Likert-based dimensions for reasoning quality (overall reasoning, relevance, interpretation, personalization, domain knowledge, logic, harm avoidance, clarity), 6 dimensions for code quality, plus error and recovery rates."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Extensive human evaluation: 650 hours, 19 annotators (12 for reasoning quality, 7 domain experts for code quality), plus qualitative interviews with 4 raters (Section 4.2, 4.4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 states the 172 open-ended queries 'were intentionally excluded from agent development to avoid potential over-fitting.' Few-shot examples were selected via K-means clustering on a separate set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides per-query-type breakdown of PHIA's improvement over Code Generation across 9 query types (Correlation, General Knowledge, Problematic, etc.). Figure 5 breaks down error categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Error analysis in Section 4.3 with Figure 5 (code error categories), Figure 6 (error and recovery rates). Figure D.1 shows a low-scoring PHIA output. Qualitative analysis in Section 4.4 discusses failure modes. Supplementary Figures B.3, G.2, G.5 show error recovery and failed attempts."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "PH-LLM 'is not able to answer any of our objective queries' (Section 4.3). PHIA showed no significant improvement over Code Generation for personalization and harm avoidance dimensions. Personal Min/Max/Avg queries showed 'effectively zero' improvement (Section 4.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: '84% accuracy on objective, numerical questions' matches Figure 3-A (84.2%). '83% favorable ratings' matches Section 4.3 ('83% of PHIA's responses as Fair or better'). 'Twice as likely to achieve the highest quality rating' matches Figure A.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'PHIA's ability to strategically plan at the first Thought step and perform iterative reasoning about its outputs through the remaining Thought steps minimizes error-prone code generation' (Section 4.3). The comparison between Code Generation and PHIA involves multiple simultaneous differences (iterative reasoning, thought steps, web search). Only search is ablated; the contribution of iterative reasoning vs. thought planning is not isolated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Transforming Wearable Data into Personal Health Insights' is broad but testing is limited to Gemini 1.0 Ultra only, 4 synthetic users from Fitbit/Pixel Watch, 31-day data windows. Abstract claims 'enabling a new era of accessible, personalized, and data-driven wellness for the wider population' far exceed what was tested. Section 7 acknowledges single-model limitation but title/abstract remain unbounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for PHIA's superior performance. Could the improvement be due to simply having more tokens/compute per query rather than the agent framework specifically? Could the few-shot examples for PHIA be higher quality than those for Code Generation? These alternatives are not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: 'we make no claim as to the effectiveness of these insights for helping real users understand their data, facilitating behavior changes, and ultimately improving health outcomes. Our aim in this paper is to define methods, tasks, and evaluation frameworks.' This clearly distinguishes the proxy (accuracy/ratings) from the ultimate outcome (health improvement)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Gemini 1.0 Ultra' (Section 3) and 'GPT-4' (Section 4.1) without snapshot dates or API versions. Per schema, marketing names without snapshot dates do not count as specified versions. Supplement I mentions 'Gemini 1.5 Pro' for validation, also without version specifics."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Extensive few-shot examples are provided in Supplement E (Figures E.1-E.7) showing the actual ReAct trajectories used for PHIA and the corresponding Code Generation and Numerical Reasoning examples. These appear to be the actual text used in prompting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, max tokens, or any LLM sampling hyperparameters for Gemini 1.0 Ultra or GPT-4. K=20 for clustering few-shot examples is mentioned, but LLM API parameters are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The ReAct agent framework is described in detail (Section 3): Thought/Act/Observe cycle, Python sandbox runtime with Pandas, Google Search API integration, few-shot example selection via sentence-T5 embeddings and K-means clustering. Figure 1 provides a workflow diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Template-based query generation is described (Section 2.1), including the custom during() function. Synthetic data generation pipeline is detailed (Section 2.3): CPAR neural network, Gaussian Copula model, missing data patterns. Open-ended query filtering: 3000 → 200 (random sample) → 172 (after similarity exclusion)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations and Future Work' is extensive with six distinct subsections covering effectiveness, veracity, tool use, subjective thresholds, wearable data focus, and model generalization."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: annotators lack health expertise ('we did not employ health experts to assess the domain-specific validity'), single model limitation ('we restrict our experiments to a single base language model'), translation pipeline noise ('the language model based translation process... may introduce noise'), limited synthetic users ('we aggregated user data over 31-day periods with a minimum of 10 days')."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: 'PHIA and similar systems should not be employed to derive insights into conditions that cannot be accurately assessed using wearable devices,' 'the scope of this study is deliberately limited to conditions that can be monitored with consumer wearables,' and 'we make no claim as to the effectiveness of these insights for... facilitating behavior changes.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases synthetic wearable user data and the evaluation dataset: 'Our data and code are available at this link' (Section 1). The synthetic data enables independent verification of the objective query results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described for all three datasets: objective queries via expert-crafted templates (Section 2.1), open-ended queries via colleague survey (Section 2.2), synthetic wearable data from 30000 real users with CPAR neural network (Section 2.3). Time period (October 2023), inclusion criteria (10+ days, age 18-80), and device types are specified."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Wearable users: 'randomly selected from individuals with heart rate-enabled Google Fitbit and Google Pixel Watch devices' with IRB approval and informed consent (Sections 2.3, 11). Annotators: 'recruited a team of twelve independent annotators... hailing from Kenya, China, India, and the United States' with specific qualifications (Supplement G.5). Query contributors: 'A survey was conducted with a sample of the authors' colleagues' (Section 2.2)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from collection to analysis is documented: 30000 real users → CPAR synthetic generation → 56 synthetic users → 4 selected for evaluation. Query pipeline: templates → 4000 objective queries; colleague survey → 3000 queries → 200 sampled → 172 after similarity filtering. Each step and filtering criterion is explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 10: 'This study was funded by Google Research.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with '1Google Research' affiliation. The author list header states the Google Research affiliation clearly."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google funded the study and all authors are Google/Alphabet employees. Google owns Fitbit (the wearable platform) and Gemini (the LLM). Google has direct commercial interest in demonstrating that their LLM can add value to their wearable ecosystem."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "Section 10: 'All authors are or were employees of Alphabet and may own stock as part of the standard compensation package.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of Gemini 1.0 Ultra's training data cutoff date. Although the benchmark uses synthetic data (reducing contamination risk), the model's training cutoff is not stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether similar wearable data analysis tasks or health Q&A formats appeared in Gemini's training data. The synthetic data mitigates direct overlap, but this is not explicitly discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The custom benchmark uses synthetic data generated after model training, which inherently reduces contamination risk, but the paper does not discuss this advantage or address contamination concerns explicitly."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration for the evaluation study. No OSF, AsPredicted, or other pre-registration links."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section 11: 'This study was conducted with the approval of an independent Institutional Review Board (IRB), ensuring compliance with ethical guidelines for research involving human data. All participants provided informed consent.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Annotator demographics reported in Supplement G.5: nationalities (Kenya, China, India, US), degree fields (education, information systems, digital arts, statistics, economics). Expert demographics: 'seven data scientists with graduate degrees, extensive professional experience... mean = 9 years' (Section 4.2). Wearable users: age 18-80."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Wearable users: 'at least 10 days of data recorded during October 2023, with a profile age between 18 and 80 years old' (Section 2.3). Annotators: 'significant prior exposure to projects focusing on LLM-based health queries and high proficiency in English' (Supplement G.5)."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with participant randomization to conditions. Annotators rated blinded model outputs in a within-subjects design rather than being assigned to treatment/control conditions."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2: 'All responses were distributed so that each was rated by at least three unique annotators, who were blinded to the method used to generate the response.' Also for code quality: 'Experts were blinded to the experimental condition.'"
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether any annotators or experts dropped out of the evaluation. Started with 12 reasoning annotators and 7 code experts; final numbers are implied to be the same but attrition is not explicitly reported."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of API costs, tokens consumed, or inference latency per query for PHIA or any baseline. PHIA's multi-step reasoning involves multiple LLM calls and web searches per query, but cost is not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of total computational budget. 650 hours of human evaluation time is stated, but LLM compute (total API calls across 4000 objective + 172 open-ended queries × multiple methods) is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or sensitivity analysis. Results appear to be from single runs. LLM sampling stochasticity is not addressed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It is unclear whether the 4000 objective queries and 172 open-ended queries were run once or multiple times per method."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The choice of K=20 for few-shot clustering is stated but no search over alternative configurations is described. Temperature and other LLM parameters are not even reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The few-shot selection process (K-means with K=20) is described but not justified. No comparison of alternative few-shot selection strategies, different K values, or other configuration choices."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple Wilcoxon signed-rank tests are performed across 8 reasoning dimensions and 6 code quality dimensions (Figures 3-B, 3-C) without any mention of Bonferroni, Holm, or other multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Google employees evaluate their own Gemini-based system against baselines they constructed. The Code Generation baseline and Numerical Reasoning baseline were designed by the authors. The bias of evaluating one's own system is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "PHIA makes multiple LLM calls per query (Thought + Code + Search + Observe cycles) while Code Generation makes one. This compute difference is not discussed or controlled for. The improvement could partly reflect more compute rather than better architecture."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether their benchmark actually measures the claimed capability of 'personal health insights.' The objective queries test numerical extraction, not health insight quality. The open-ended queries are from colleagues rather than real wearable users, and the Likert ratings may not correlate with actual health value."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The scaffold (ReAct agent framework) IS the thing being tested — the paper evaluates whether an agentic framework improves over non-agentic approaches. The paper does not compare different models; it uses the same Gemini 1.0 Ultra throughout."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The synthetic data is generated from October 2023 wearable data, and the model's training data may include similar wearable health Q&A patterns, but this is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The agent has access to the full user data table during evaluation, which is appropriate for the task, but the potential for the LLM to leverage patterns from training rather than performing genuine analysis is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data. The template-generated queries may resemble patterns in LLM training data (e.g., common data analysis questions). Not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. The use of synthetic data provides some natural protection, but no formal detection method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "PHIA achieves 84% exact match accuracy on objective personal health queries, significantly outperforming Code Generation (74%), Numerical Reasoning (22%), and GPT-4 chain-of-thought (53.6%).",
       "evidence": "Figure 3-A shows PHIA at 84.2% with 95% bootstrapped confidence intervals. All 4000 objective queries evaluated (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "PHIA demonstrates superior reasoning on open-ended queries, with 83% of responses rated 'Fair' or better and overall reasoning score of 68 vs 52 for Code Generation.",
       "evidence": "Figure 3-B shows significant Wilcoxon signed-rank test results (p < 0.05) across 6 of 8 dimensions. 650 hours of human evaluation, 5500+ responses, 3 annotators per response (Sections 4.2, 4.3).",
       "supported": "strong"
     },
     {
       "claim": "PHIA is twice as likely to generate 'Excellent' responses compared to Code Generation.",
       "evidence": "Figure A.2 shows the distribution of quality ratings with PHIA having approximately double the 'Excellent' fraction.",
       "supported": "moderate"
     },
     {
       "claim": "PHIA's error rate is half that of Code Generation (0.192 vs 0.395), and PHIA recovers from errors 11.4% of the time vs 0% for Code Generation.",
       "evidence": "Figure 6 shows error and recovery rates with 95% bootstrapped confidence intervals. Expert annotators categorized errors in Figure 5 (Section 4.3).",
       "supported": "strong"
     },
     {
       "claim": "PHIA's iterative reasoning and tool use capability produces substantially higher quality results than the same base language model without the agent framework.",
       "evidence": "All comparisons use Gemini 1.0 Ultra as the base model. PHIA outperforms Code Generation on most metrics (Figures 3-B, 3-C). Search ablation in Figure A.1 shows domain knowledge improvement comes from search specifically.",
       "supported": "moderate"
     },
     {
       "claim": "Over 99% of PHIA's responses are rated as harmless by human annotators.",
       "evidence": "Section 4.3 states 'over 99% of responses rated as harmless.' Harm avoidance scores in Figure 3-B are saturated near perfect. Examples of safe refusals shown in Supplement C.",
       "supported": "moderate"
     },
     {
       "claim": "PH-LLM is unable to answer any of the objective health queries due to its fine-tuning on aggregated coaching data only.",
       "evidence": "Section 4.3: 'PH-LLM model is not able to answer any of our objective queries due to its limitations in handling detailed, long-context tabular data inputs after being fine-tuned exclusively on aggregated coaching case study data.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "PHIA, an LLM agent using iterative ReAct reasoning with code generation and web search, achieves 84% accuracy on objective health queries (vs 74% for code generation alone and 22% for numerical reasoning). On open-ended queries, PHIA earns significantly higher human ratings across 6 of 8 quality dimensions, with the largest gains in domain knowledge and general knowledge query types. PHIA makes half as many code errors as the baseline and recovers from errors 11.4% of the time. The agent framework's benefits stem from iterative reasoning and web search integration, though both methods performed similarly on personalization and harm avoidance.",
   "red_flags": [
     {
       "flag": "Company evaluating own product ecosystem",
       "detail": "All authors are Google Research employees/affiliates. Google owns both Fitbit (the wearable data source) and Gemini (the LLM). The study demonstrates commercial value of combining Google's wearable data platform with Google's LLM. This conflict is disclosed (Section 10) but creates systematic incentive to show positive results."
     },
     {
       "flag": "Extremely small user sample",
       "detail": "Only 4 synthetic wearable users were used for evaluation, randomly selected from 56 generated users. This tiny sample cannot represent the diversity of real-world wearable usage patterns, health conditions, or data quality issues. Results may not generalize to the broader population."
     },
     {
       "flag": "Non-independent annotator recruitment",
       "detail": "Open-ended queries were sourced from 'authors' colleagues' with 'relevant expertise in personal and consumer health research.' Code evaluation experts were 'affiliated with the same institution as the authors' (Supplement G.5). This creates potential familiarity and expectation biases in both the evaluation dataset and the ratings."
     },
     {
       "flag": "Translation pipeline introduces uncontrolled noise",
       "detail": "For reasoning evaluation, Python code was translated to English using Gemini Ultra itself (Supplement G.2) because annotators lacked programming skills. This LLM-based translation adds an uncontrolled noise source — if the translation is more favorable for PHIA's multi-step outputs, it would bias the comparison."
     },
     {
       "flag": "Compute confound not addressed",
       "detail": "PHIA makes multiple LLM calls per query (multiple Thought/Act/Observe cycles) while Code Generation makes one. The performance improvement could partially reflect more compute/tokens rather than the agent architecture, but this confound is never discussed or controlled for."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "Multiple Wilcoxon signed-rank tests are conducted across 14+ dimensions (8 reasoning + 6 code quality) without Bonferroni or other corrections for family-wise error rate. Some significant results at p < 0.05 may be false positives."
     }
   ],
   "cited_papers": [
     {
       "title": "React: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2023,
       "relevance": "Core agent framework used by PHIA; foundational work on LLM-based agentic reasoning with tool use."
     },
     {
       "title": "Towards a personal health large language model",
       "authors": ["Justin Cosentino", "Anastasiya Belyaeva", "Xin Liu"],
       "year": 2024,
       "arxiv_id": "2406.06474",
       "relevance": "Key baseline: fine-tuned Gemini for health coaching on wearable data; demonstrates limitations of non-agentic LLM approaches for health queries."
     },
     {
       "title": "From classification to clinical insights: Towards analyzing and reasoning about mobile and behavioral health data with large language models",
       "authors": ["Zachary Englhardt", "Chengqian Ma", "Margaret E Morris"],
       "year": 2024,
       "relevance": "Key baseline: chain-of-thought prompting of GPT-4 for wearable health data interpretation; compared against PHIA."
     },
     {
       "title": "Sparks of artificial general intelligence: Early experiments with gpt-4",
       "authors": ["Sébastien Bubeck", "Varun Chandrasekaran", "Ronen Eldan"],
       "year": 2023,
       "relevance": "Documents LLM capabilities and limitations in reasoning and tool use, motivating the agent-based approach."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive nlp tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG work that informs PHIA's web search integration for health domain knowledge retrieval."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2024,
       "relevance": "Demonstrates LLM tool-use capabilities that underpin the agent framework approach."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Major LLM agent benchmark; cited to support claim that frontier models are capable of agentic tasks."
     },
     {
       "title": "VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks",
       "authors": ["Jing Yu Koh", "Robert Lo", "Lawrence Jang"],
       "year": 2024,
       "relevance": "Multimodal agent evaluation benchmark; cited to support generalizability of agent frameworks across models."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Rigorous evaluation of LLM code generation capabilities, relevant to PHIA's code generation component."
     },
     {
       "title": "EHRAgent: Code empowers large language models for complex tabular reasoning on electronic health records",
       "authors": ["Wenqi Shi", "Ran Xu", "Yuchen Zhuang"],
       "year": 2024,
       "arxiv_id": "2401.07128",
       "relevance": "Code-writing agents for medical records analysis; closely related approach applied to health data."
     },
     {
       "title": "Gemini: a family of highly capable multimodal models",
       "authors": ["Gemini Team", "Rohan Anil", "Sebastian Borgeaud"],
       "year": 2023,
       "arxiv_id": "2312.11805",
       "relevance": "The base language model (Gemini 1.0 Ultra) used by PHIA and baselines."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational few-shot prompting work; technique used to guide PHIA's task execution."
     },
     {
       "title": "ToolQA: A dataset for LLM question answering with external tools",
       "authors": ["Yuchen Zhuang", "Yue Yu", "Kuan Wang"],
       "year": 2024,
       "relevance": "Benchmark for LLM tool-augmented question answering; relevant evaluation methodology for tool-using agents."
     }
   ]
 }