ai-research-survey

scan-v5.json (28236B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "FlockVote: LLM-Empowered Agent-Based Modeling for Simulating U.S. Presidential Elections",
     "authors": [
       "Lingfeng Zhou",
       "Yi Xu",
       "Zhenyu Wang",
       "Dequan Wang"
     ],
     "year": 2025,
     "venue": "arXiv.org (ICAIS 2025)",
     "arxiv_id": "2512.05982",
     "doi": "10.48550/arXiv.2512.05982"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'high fidelity' and 'successful replication' of the election based on getting 6/7 swing states correct in a single trial with a post-hoc selected model; the sensitivity analysis in Section 4.5 shows Democratic support swinging 22pp based on trivial prompt changes, which directly contradicts the fidelity claim.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The ablation studies (Tables 1–2) use the 2020 election as ground truth, which the paper itself acknowledges carries 'significant risk of data leakage' since LLMs may recall known outcomes rather than reason; causal attribution of prediction improvement to education/religion dimensions is therefore unreliable.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion calls for applying the framework 'beyond politics to other high-stakes domains such as economics, law, and medicine' based on a single election test; the sensitivity analysis showing 22pp swings from prompt rephrasing severely limits any generalization claim.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not seriously consider that the 6/7 correct predictions could be explained by model training on post-election coverage (data leakage): Qwen-Max-2024-04-28 has an April 2024 cutoff that precedes the election, but other model cutoffs are unstated, and no analysis distinguishes recall from reasoning.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "LLM probability outputs are treated throughout as valid proxies for real voter preferences with no discussion of the gap between a model's token probabilities and actual human decision-making; the framework conflates simulation fidelity with behavioral validity.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section; the conclusion mentions 'key challenges' in one sentence, and the sensitivity analysis is framed as a positive contribution rather than a limitations disclosure.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The data leakage threat (LLMs trained on post-election data) is acknowledged in Section 1 but never quantified or controlled for; no analysis distinguishes whether correct predictions stem from demographic reasoning or training-data recall.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state what the results do NOT show; the framework is validated on one election in one country with one primary model, yet no explicit scope boundary limits claims of generalizability.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (Shanghai Jiao Tong University, Shanghai Innovation Institute, Shanghai Academy of Social Sciences, Nanjing University) are disclosed on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms — LLM agent, agent-based modeling, computational laboratory, demographic profiling, probabilistic voting — are defined or described with sufficient context in Section 3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames its contribution as a framework for interpretable LLM-based election simulation that goes beyond predictive accuracy to audit bias and instability (Section 2 research gap paragraph).",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 explicitly contrasts FlockVote with traditional ABM, statistical models, and concurrent LLM-based election simulation work (Yu et al., Jiang et al., Bradshaw et al.), situating its novelty around interpretability and reliability auditing.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "A GitHub repository is provided (https://github.com/maple-zhou/FlockVote) with explicit mention of releasing the codebase in Appendix J.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "Input demographic data relies on public ACS/ASARB sources, but the simulated agent-level outputs, aggregated results, and the processed agent profile datasets are not released.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or dependency specification is mentioned; the only hardware reference is 'M3 MacBook Pro' as an anecdote about consumer efficiency.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Full prompts are in appendices and code is released, but there are no step-by-step instructions for running the pipeline, including API setup, data preprocessing, or how to combine ACS/ASARB data into agent profiles.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "The main results (6/7 states correct, state-level support percentages) are reported as point estimates with no CIs or error bars; only Figure 4's population-size ablation shows trial variance.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to any comparative claims, including the ablation results in Tables 1–2 or the model comparison in Table 5.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": false,
           "justification": "Percentage differences are shown (e.g., education dimension corrects Wisconsin winner) but without baseline context, confidence intervals, or standardized effect sizes.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": true,
           "justification": "Figure 4 empirically tests 10–2000 agents with 10 trials each and finds variance stabilizes at 300, justifying the 1,000-agent choice for final runs.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Variance is shown only in Figure 4's population ablation; all main results (Table 5, state predictions) are single-run point estimates with no variance reported.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": false,
           "justification": "The only comparison is to the actual election outcome (ground truth), not to traditional ABM baselines, polling averages, or statistical forecasting models that are described in the related work.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": false,
           "justification": "No external baseline models are evaluated; the concurrent work of Yu et al. and Jiang et al. is cited but not benchmarked against.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Sections 4.4.1 and 4.4.2 provide ablation studies on agent population size and demographic attribute selection (education, religion dimensions).",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": false,
           "justification": "Evaluation is limited to winner prediction accuracy (6/7 states) and raw support percentage; no calibration score, Brier score, or other probabilistic accuracy metrics are used.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "No human evaluation of system outputs is conducted; the 'interviews' are LLM self-reports, not human assessment of output quality.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "The 2024 election results serve as a held-out test case; the 2020 election is used only for ablations with an acknowledged data leakage caveat.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down per swing state for both main results and the full model comparison in Table 5 (7 states × 10 models).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Nevada (the one misclassified state), swing agents that flip votes based on JSON key ordering, and models with severe Democratic bias (Qwen-Max-09-19) are explicitly analyzed as failure modes.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.5 explicitly reports that agents are 'extraordinarily sensitive' to prompt phrasing (22pp swing) and show positional instability — framed as findings rather than buried.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A lists exact versioned model IDs for all models used (e.g., Qwen-Max-2024-04-28, GPT-4o-2024-08-06, Claude-3-5-sonnet-2024-10-22, Gemini-1.5-Pro-002).",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "All 8 context variants, the bias evaluation prompts, the full voting prompt, the system prompt for mitigation, and the interactive interview prompt are reproduced verbatim in appendices C–I.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Temperature is stated: 0.7 for the main results run (diversity and realism) and 0 for reliability/stability experiments (Section 4.1).",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The three-step construction (demographic profiling, contextual information injection, probabilistic JSON output) and the aggregation procedure are described in Section 3.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "ACS and ASARB sources are cited but the preprocessing steps — how joint vs. independent distributions were constructed, how missing cells were handled — are not documented beyond Appendix B's dimension tables.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Individual agent responses and the full simulation outputs are not released; only aggregate percentages are reported in tables and figures.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The two demographic data sources (2023 ACS and 2020 ASARB Religion Census) are identified with URLs, and the eight attribute dimensions derived from them are listed in Appendix B.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants — agents are synthetically generated from census distributions.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "The high-level pipeline is described (census data → profile sampling → LLM query → probability aggregation) but key steps such as how joint distributions were estimated, how religion data was merged with ACS data, and how candidate policy summaries were constructed are not documented.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training cutoffs are not stated for any of the tested models; only Qwen-Max-2024-04-28's name implies an April 2024 cutoff, but this is not confirmed and other models' cutoffs are unstated.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper argues the 2024 election prevents data leakage (Section 1), but does not verify whether any models' training corpora include post-election reporting, nor does it test whether models can recall exact state-level results.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "While the paper motivates using 2024 to avoid leakage, no empirical test is run (e.g., directly asking models who won each swing state) to verify that the models cannot recall the election outcome from training data.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix J reports approximately 160k tokens per state for the optimized prompt and notes that Llama3.2-3B on an M3 MacBook completes predictions in one hour.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total compute budget (API calls, total tokens across all experiments, cost in dollars) is stated for the full set of experiments across 7 states and 10+ models.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "FlockVote correctly replicates the macro-level outcome of the 2024 US Presidential Election, predicting Trump wins in 6 of 7 swing states",
       "evidence": "Figure 2 compares predicted vs. actual results; Nevada is the only discrepancy (predicted Democratic win by 0.17% margin vs. actual Republican win)",
       "supported": "moderate"
     },
     {
       "claim": "LLM agents exhibit severe political bias — most models default to pro-Democratic predictions even under prompts designed to favor Trump",
       "evidence": "Table 3 shows Qwen-Max-09-19 predicts Democratic victory in Georgia even under 'Asymmetric Positive Framing for Trump' condition",
       "supported": "strong"
     },
     {
       "claim": "Agent predictions are extraordinarily sensitive to semantically irrelevant prompt changes, with Democratic support ranging from 36.2% to 58.6% across 8 minimal rephrasing variants",
       "evidence": "Figure 7 shows support rate swings across 8 context variants in Pennsylvania using Qwen-Max-04-28",
       "supported": "strong"
     },
     {
       "claim": "Including education and religion demographic dimensions meaningfully improves simulation accuracy",
       "evidence": "Tables 1–2 show 6-dimension model fails Wisconsin winner prediction; adding education corrects this; religion reduces Democratic bias",
       "supported": "weak"
     },
     {
       "claim": "A simulation of 300 agents per state achieves stable predictions",
       "evidence": "Figure 4 shows variance stabilizes at 300 agents across 10 repeated trials with different random seeds",
       "supported": "strong"
     },
     {
       "claim": "Positional order of candidates in the JSON response format causes agents to completely flip their vote",
       "evidence": "Appendix H documents 3 'Swing Agents' whose preference inverts solely when candidate key order in the JSON schema changes",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "observational",
     "case-study",
     "benchmark-eval"
   ],
   "key_findings": "FlockVote uses LLM agents with demographic profiles to simulate the 2024 US election, correctly predicting 6 of 7 swing state winners. However, the framework's core reliability findings are more significant than the prediction result: agents exhibit severe political bias (most models are pro-Democratic by default), produce support rates that swing 22 percentage points from trivial prompt rephrasing, and completely invert voting preference when candidate names are reordered in JSON output. These instabilities undermine the framework's use as a reliable social science instrument despite its surface-level predictive success.",
   "red_flags": [
     {
       "flag": "Post-hoc model selection",
       "detail": "The primary model (Qwen-Max-2024-04-28) was chosen because it produced 'more neutrality' and the correct result; the paper admits this choice was 'fortuitous.' Other tested models show severe Democratic bias, meaning the success depends entirely on this specific model version."
     },
     {
       "flag": "Data leakage unverified",
       "detail": "The paper argues using the 2024 election avoids leakage, but does not verify training cutoffs for most models or test whether models can recall election outcomes directly. The correct prediction could be recall rather than reasoning."
     },
     {
       "flag": "No non-LLM baselines",
       "detail": "The framework is never compared against polling averages, statistical forecasting models, or traditional ABMs — the methods it claims to surpass. The only comparison is to the ground truth outcome."
     },
     {
       "flag": "Single election, massive generalization",
       "detail": "All empirical results come from the 2024 US presidential election (7 swing states), yet the conclusion advocates applying the framework to economics, law, and medicine."
     },
     {
       "flag": "Sensitivity analysis contradicts fidelity claim",
       "detail": "The paper simultaneously claims 'high fidelity' in the abstract and demonstrates 22pp support swings from trivial prompt changes in Section 4.5; these claims are not reconciled."
     },
     {
       "flag": "No statistical testing",
       "detail": "No significance tests, confidence intervals, or effect sizes are reported for any comparative claims, including ablation results and model comparisons."
     }
   ],
   "cited_papers": [
     {
       "title": "Out of one, many: Using language models to simulate human samples",
       "relevance": "Foundational work on LLMs as human simulators (Argyle et al., 2023) that FlockVote directly builds upon"
     },
     {
       "title": "Generative Agents: Interactive Simulacra of Human Behavior",
       "relevance": "Park et al. 2023 Stanford 'small town' paper — the key prior LLM-based ABM work that FlockVote extends to political simulation"
     },
     {
       "title": "Simulating human behavior with AI agents",
       "relevance": "Park et al. 2024 '1,000 people simulation' replicating survey responses — directly validates LLM agent approach used here"
     },
     {
       "title": "Hidden persuaders: LLMs' political leaning and their influence on voters",
       "relevance": "Potter et al. 2024 — cited for evidence that biased LLM agents influence real voter opinions, motivating FlockVote's reliability audit focus"
     },
     {
       "title": "LLM stability: A detailed analysis with some surprises",
       "relevance": "Atil et al. 2024 — cited for evidence of LLM non-determinism even at zero temperature, supporting FlockVote's instability findings"
     },
     {
       "title": "A large-scale empirical study on large language models for election prediction",
       "relevance": "Yu et al. 2024 — concurrent work on LLM-based election simulation that FlockVote is compared against in the related work"
     },
     {
       "title": "Donald Trumps in the virtual polls: Simulating and predicting public opinions in surveys using large language models",
       "relevance": "Jiang et al. 2024 — concurrent election simulation work using persona-based micro-simulation, directly comparable to FlockVote"
     },
     {
       "title": "Probing LLM Prompt Sensitivity (ProSA)",
       "relevance": "Zhuo et al. 2024 — cited for evidence that LLMs are sensitive to semantically equivalent prompt changes, motivating stability analysis"
     },
     {
       "title": "Benchmarking distributional alignment of large language models",
       "relevance": "Meister et al. 2025 — cited to justify probabilistic voting output format as more accurate and stable than binary choice"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The framework is released as open-source code and runs on consumer hardware, but the instability findings undermine practical deployment for real forecasting."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that JSON key ordering alone causes agents to completely flip their vote is genuinely surprising and challenges LLM reliability assumptions for simulation."
     },
     "fear_safety": {
       "score": 2,
       "justification": "The paper explicitly cites evidence that biased LLM agents actually change real voters' opinions, framing AI election simulation as a social safety issue."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "US presidential election context is inherently high-drama; sensitivity analysis showing models systematically favor Democrats regardless of prompting adds controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is released on GitHub and runs on consumer hardware in one hour with Llama3.2, making it readily demonstrable."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Shanghai Jiao Tong University is a well-known institution but not a top AI lab; GPT-4o, Claude, Gemini are named in experiments which adds recognition."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "10762409",
         "title": "Scientific publications should be anonymous",
         "points": 128,
         "comments": 76,
         "url": "https://news.ycombinator.com/item?id=10762409",
         "created_at": "2015-12-19T02:50:25Z"
       },
       {
         "hn_id": "31318574",
         "title": "Flares from black hole binaries: black hole shadows via light-curve tomography",
         "points": 43,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=31318574",
         "created_at": "2022-05-09T19:24:38Z"
       },
       {
         "hn_id": "29549353",
         "title": "Self-attention Does Not Need O(n^2) Memory",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=29549353",
         "created_at": "2021-12-14T08:01:16Z"
       },
       {
         "hn_id": "25405164",
         "title": "Emergent Quantumness in Neural Networks",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=25405164",
         "created_at": "2020-12-13T08:33:43Z"
       },
       {
         "hn_id": "46720522",
         "title": "Accurate and efficient thermal modeling for 2.5D/3D heterogeneous chiplets",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46720522",
         "created_at": "2026-01-22T15:29:20Z"
       },
       {
         "hn_id": "47240426",
         "title": "Learning-Based Multi-Stage Strategy for Aircraft to Evade Missile",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47240426",
         "created_at": "2026-03-03T23:09:24Z"
       },
       {
         "hn_id": "29576916",
         "title": "Self-Attention does not need O(n^2) Memory",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=29576916",
         "created_at": "2021-12-16T10:35:59Z"
       }
     ],
     "top_points": 128,
     "total_points": 180,
     "total_comments": 77
   }
 }