ai-research-survey

scan.json (25949B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "FlockVote: LLM-Empowered Agent-Based Modeling for Simulating U.S. Presidential Elections",
     "authors": ["Lingfeng Zhou", "Yi Xu", "Zhenyu Wang", "Dequan Wang"],
     "year": 2025,
     "venue": "ICAIS 2025",
     "arxiv_id": "2512.05982",
     "doi": "10.48550/arXiv.2512.05982"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/maple-zhou/FlockVote (footnote 1, also Appendix J)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Demographic data sourced from publicly available 2023 ACS and 2020 ASARB datasets. Code release includes the framework."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment/dependency specifications mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. Appendix J mentions code release but no explicit instructions for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows error bars across 10 trials with different random seeds for the agent population stability analysis."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests used. Comparisons between models (Table 3, Figure 6) and context variants (Figure 7) are based on point estimates with no statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Support rate percentages with baselines are reported throughout (e.g., 'Democrats win Nevada with a margin of only 0.17%', full tables with Republican/Democrat percentages in Table 5)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4.1 explicitly validates agent population size through stability analysis (10 to 2000 agents, 10 trials each), finding stabilization at 300 agents."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows variance across 10 trials with distinct random seeds for different agent population sizes."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Compares against actual 2024 election results (Figure 2) and tests 7 different LLMs (Table 5, Figure 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Uses contemporary models: GPT-4o-2024-08-06, Claude-3-5-sonnet-2024-10-22, Gemini-1.5-Pro-002, DeepSeek-V2.5 (Appendix A)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 ablates agent population size (4.4.1) and profile dimensions — education (Table 1) and religion (Table 2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregated support rate percentages are used. No additional metrics such as calibration, Brier score, or demographic-level accuracy metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the simulation outputs. The 'interviews' in Section 4.3 are with LLM agents, not human judges evaluating output quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Ablation studies use 2020 election data (Table 1, 2) and main results use 2024, but model selection (Qwen-Max-04-28 identified as best via Table 3) was not separated from 2024 evaluation. No explicit dev/test separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by state (7 swing states), by model (Table 5), by context condition (Table 3), and by demographic group (Figure 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 extensively discusses failures: political bias in models, context sensitivity causing wild fluctuations (36.2% to 58.6%), positional instability ('swing agents'), and model disagreements."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports that agents are 'flawed, non-deterministic instruments', that minor prompt rephrasing causes wild fluctuations, and that candidate ordering alone flips votes. Appendix I reports that their mitigation strategy shows only 'minor improvements'."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about replicating the real-world outcome are supported by Figure 2 (6/7 swing states correct). Claims about interpretability are demonstrated in Section 4.3. Claims about sensitivity analysis are in Section 4.5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies causal mechanisms — that demographic profiles and policy context 'enable nuanced generative reasoning' to simulate voting. Ablation studies (Section 4.4) support some component contributions, but the core causal claim that LLM agents replicate voter reasoning (vs. pattern matching on training data) is not justified."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title claims 'Simulating U.S. Presidential Elections' but tests only 7 swing states in one election. Conclusion calls for application to 'economics, law, and medicine.' The framework's success is demonstrated for one specific election with one primary model, yet generalization claims are broad."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 extensively discusses alternative explanations: model political bias (Table 3), prompt sensitivity (Figure 7), positional instability (swing agents), and questions whether 'these agents are valid tools for social science.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures LLM probability outputs and frames this as 'simulating voter decisions.' It does not adequately distinguish between LLM text generation based on training data patterns and actual voter decision-making processes. The possibility that correct predictions reflect training data correlations rather than reasoning is not addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A lists specific versions: Qwen-Max-2024-04-28, GPT-4o-2024-08-06, Claude-3-5-sonnet-2024-10-22, Gemini-1.5-Pro-002, etc."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts provided in Appendix C (voting prompt), Appendix E (bias experiment prompts), Appendix G (context variants), and Appendix I (mitigation system prompt)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'All experiments use a temperature of 0 for stability, except the main result (0.7 for diversity and realism).'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Each agent is a single prompt-response call to an LLM API."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes how demographic profiles are generated from ACS and ASARB data using joint and independent distributions, with 1000 agents per state via random sampling."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The conclusion (Section 5) briefly mentions 'key challenges regarding agent bias and instability' in one sentence. Section 4.5 functions as limitation analysis but is framed as 'Sensitivity Analysis.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 identifies specific threats: pro-Democratic model bias (Table 3), context variant sensitivity causing 22+ percentage point swings (Figure 7), positional instability in JSON response format (swing agents), and model-to-model variation (Figure 6)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The conclusion calls for application to 'economics, law, and medicine' without bounding the current findings to the specific election, models, and demographic framework tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Code released on GitHub. Demographic data from publicly available ACS and ASARB datasets. Agent profiles are generated programmatically from these sources."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data sources (2023 ACS, 2020 ASARB), the eight demographic attributes used, and how joint/independent distributions are applied. Section 3.2 describes contextual information sources (Pew, Gallup, NBC)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Agents are synthetically generated from demographic distributions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full pipeline: demographic modeling (3.1) → contextual information (3.2) → probabilistic voting behavior (3.3), with demographic categories detailed in Appendix B."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Shanghai Jiao Tong University, Shanghai Innovation Institute, Shanghai Academy of Social Sciences, Nanjing University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed at all, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper argues the 2024 election is contemporary enough to prevent data leakage (Section 1, 4.1) but never states the actual training data cutoff dates for Qwen-Max-2024-04-28 or any other model used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 and 4.1 explicitly discuss data leakage risk: 'historical events like the 2020 election present a significant risk of data leakage, where LLMs might simply recall known outcomes rather than reasoning dynamically.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The 2024 election was explicitly chosen as the testbed because it is 'a contemporary event that prevents data leakage from LLM training data' (Section 4.1). The 2020 election contamination risk is acknowledged."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All agents are LLM-based synthetic simulations."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Synthetic agent demographics are reported in Appendix B."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix J: 'reducing token consumption to approximately 160k tokens per state' and 'accurate predictions can be produced with Llama3.2-3B-Instruct in only one hour' on a consumer-grade device."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix J states 'even on a consumer-grade device (M3 MacBook Pro), accurate predictions can be produced... in only one hour.' Token consumption of ~160k tokens per state is also stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows results across 10 trials with distinct random seeds for agent population sizes from 10 to 2000."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4.1: 'each agent number repeated over 10 trials using distinct random seeds to generate unique agent profiles.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature values (0 and 0.7) are stated but no hyperparameter search budget is reported. No discussion of how temperature values were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The primary model Qwen-Max-04-28 is described as a 'fortuitous choice' (Section 4.5) showing 'more neutrality.' This is post-hoc justification, not a systematic selection process."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the framework, chose the demographic dimensions, selected the model, and crafted the prompts, yet do not discuss author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All models are used via API calls with similar compute costs; compute differences are negligible."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 and the Related Work (Section 2.2) extensively question whether LLM agents are valid instruments for social simulation, asking 'are these agents valid tools for social science?'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. Each agent is a single prompt-response interaction."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'a contemporary event that prevents data leakage from LLM training data, thereby testing the agents' generative reasoning rather than recall.' The 2024 election was explicitly chosen to avoid temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the contextual information provided (candidate stances) or the demographic framing could leak outcome information through the model's training data associations."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether agent responses are independent. All agents use the same model, potentially sharing systematic biases. The positional instability finding (Section 4.5) hints at this but does not formally address non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method applied. The paper only uses temporal selection (choosing the 2024 election) as a prevention strategy, not a detection method."
       }
     }
   },
   "claims": [
     {
       "claim": "FlockVote correctly replicates the macro-level result that Donald Trump would win six of the seven pivotal swing states in the 2024 election.",
       "evidence": "Figure 2 compares simulation results with actual election outcomes. The only discrepancy is Nevada, predicted for Harris by 0.17% margin (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "Prediction variance stabilizes when agent population reaches 300, after which further increases yield minimal fluctuations.",
       "evidence": "Figure 4 shows stability analysis from 10 to 2000 agents across 10 trials each in Pennsylvania (Section 4.4.1).",
       "supported": "strong"
     },
     {
       "claim": "Adding education and religion dimensions to agent profiles improves simulation fidelity.",
       "evidence": "Table 1 shows education fixes the Wisconsin prediction; Table 2 shows religion improves alignment with polling data (Section 4.4.2).",
       "supported": "moderate"
     },
     {
       "claim": "LLM agents exhibit severe political bias, with most models showing a strong default pro-Democratic bias.",
       "evidence": "Table 3 shows Qwen-Max-09-19 predicts Democratic victory even under asymmetric pro-Trump framing in Georgia (Section 4.5).",
       "supported": "strong"
     },
     {
       "claim": "Agent decisions are highly sensitive to semantically irrelevant prompt changes, with Democratic support swinging from 36.2% to 58.6% across context variants.",
       "evidence": "Figure 7 shows results for 8 semantically neutral but syntactically different context variants in Pennsylvania (Section 4.5).",
       "supported": "strong"
     },
     {
       "claim": "Agents exhibit positional instability where swapping candidate order in the JSON response format causes complete vote flips.",
       "evidence": "Appendix H provides examples of 'Swing Agents' who invert their preference solely due to JSON key ordering (Section 4.5).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "FlockVote uses LLM agents with demographic profiles to simulate the 2024 U.S. presidential election across 7 swing states, correctly predicting 6 of 7 outcomes. However, the paper's most significant findings are negative: LLM agents exhibit severe political bias (pro-Democratic default), extreme sensitivity to minor prompt rephrasing (22+ percentage point swings), and positional instability where changing JSON key order flips votes. These findings challenge the reliability of LLM-based social simulation despite the macro-level accuracy.",
   "red_flags": [
     {
       "flag": "Post-hoc model selection",
       "detail": "The primary model Qwen-Max-04-28 was identified as the best choice through the same 2024 election data used for main results. The paper acknowledges this was a 'fortuitous choice' rather than a principled selection, raising concerns about overfitting to the specific outcome."
     },
     {
       "flag": "Extreme prompt sensitivity undermines main result",
       "detail": "The paper's own sensitivity analysis (Figure 7) shows that trivial prompt variations cause 22+ percentage point swings. This calls into question whether the main result's accuracy is robust or coincidental with the chosen prompt formulation."
     },
     {
       "flag": "No statistical tests for any comparisons",
       "detail": "All comparisons between models, contexts, and conditions are based on point estimates. With the demonstrated high variance across prompt variants, the absence of statistical testing makes it impossible to determine if observed differences are meaningful."
     },
     {
       "flag": "Training data contamination not ruled out",
       "detail": "While the 2024 election was chosen to avoid contamination, the paper never verifies the training cutoff dates of the models used. Qwen-Max-2024-04-28 could have been trained on polling data that predicted similar outcomes, which would explain the 'correct' predictions without genuine reasoning."
     }
   ],
   "cited_papers": [
     {
       "title": "Generative agents: Interactive simulacra of human behavior",
       "authors": ["Joon Sung Park", "Joseph O'Brien", "Carrie Jun Cai", "Meredith Ringel Morris", "Percy Liang", "Michael S Bernstein"],
       "year": 2023,
       "relevance": "Foundational work on LLM agents simulating human behavior, directly relevant to agentic AI capabilities."
     },
     {
       "title": "Out of one, many: Using language models to simulate human samples",
       "authors": ["Lisa P Argyle", "Ethan C Busby", "Nancy Fulda", "Joshua R Gubler", "Christopher Rytting", "David Wingate"],
       "year": 2023,
       "relevance": "Validated using LLM agents for social science research by replicating human survey responses."
     },
     {
       "title": "Large language models as simulated economic agents: What can we learn from homo silicus?",
       "authors": ["John J Horton"],
       "year": 2023,
       "relevance": "Proposed LLMs as simulated economic agents, foundational concept for LLM-based social simulation."
     },
     {
       "title": "Hidden persuaders: LLMs' political leaning and their influence on voters",
       "authors": ["Yujin Potter", "Shiyang Lai", "Junsol Kim", "James Evans", "Dawn Song"],
       "year": 2024,
       "arxiv_id": "2410.24190",
       "relevance": "Demonstrates that biased LLM agents can actually change real voters' opinions, relevant to AI safety and influence."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analyzes failure modes of multi-agent LLM systems including reasoning-action mismatch."
     },
     {
       "title": "ProSA: Assessing and understanding the prompt sensitivity of LLMs",
       "authors": ["Jingming Zhuo", "Songyang Zhang", "Xinyu Fang"],
       "year": 2024,
       "arxiv_id": "2410.12405",
       "relevance": "Directly relevant to understanding LLM sensitivity to prompt variations, a key finding in FlockVote."
     },
     {
       "title": "LLM stability: A detailed analysis with some surprises",
       "authors": ["Berk Atil", "Alexa Chittams", "Liseng Fu", "Ferhan Ture"],
       "year": 2024,
       "relevance": "Documents non-deterministic instability in LLMs even at zero temperature, relevant to reliability of LLM-based simulation."
     },
     {
       "title": "Measuring gender and racial biases in large language models: Intersectional evidence from automated resume evaluation",
       "authors": ["Jiafu An", "Difang Huang", "Chen Lin", "Mingzhu Tai"],
       "year": 2025,
       "relevance": "Demonstrates pervasive demographic biases in LLMs, directly relevant to bias in LLM-based voter simulation."
     },
     {
       "title": "ElectionSim: Massive population election simulation powered by large language model driven agents",
       "authors": ["Xinnong Zhang", "Jiayu Lin", "Libo Sun"],
       "year": 2024,
       "arxiv_id": "2410.20746",
       "relevance": "Concurrent work on LLM-powered election simulation, directly comparable methodology."
     },
     {
       "title": "Benchmarking distributional alignment of large language models",
       "authors": ["Nicole Meister", "Carlos Guestrin", "Tatsunori B Hashimoto"],
       "year": 2025,
       "relevance": "Validates probabilistic output format for LLM simulation tasks, methodology adopted by FlockVote."
     }
   ]
 }