ai-research-survey

scan.json (30821B)

---

 {
   "paper": {
     "title": "EcoGym: Evaluating LLMs for Long-Horizon Plan-and-Execute in Interactive Economies",
     "authors": [
       "Xavier Hu",
       "Jinxiang Xia",
       "Shengze Xu",
       "Kangqi Song",
       "Yishuo Yuan"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.09514"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "EcoGym introduces three economic simulation environments (Vending, Freelance, Operation) for evaluating LLM long-horizon planning over 365-day horizons. Evaluating 11 LLMs reveals no single model dominates across all scenarios, with performance gaps stemming from trade-offs between strategic prioritization and execution efficiency. Context window expansion does not consistently improve performance, memory module effectiveness is highly model- and task-dependent, and top-tier models surpass a sparsely-described human baseline in the Operation environment.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided on page 1: https://github.com/OPPO-PersonalAI/EcoGym."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark environments (600+ SKUs for Vending, 5k+ tasks for Freelance derived from public datasets) are released as part of the open-source repository. Freelance uses publicly available datasets (LiveCodeBench, SWE-bench, GSM8K, etc.) with mutation."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or library version information provided in the paper. Only model API versions are listed in Appendix A."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. A GitHub repository is linked but the paper contains no commands, scripts, or README-like instructions to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results table (Table 2) reports point estimates only for all three environments. Freelance and Operation results are single-run. Table 4 reports mean ± std for one secondary experiment, but the primary evaluation has no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used anywhere. All comparative claims (e.g., 'Gemini-3 series demonstrates dominant asset appreciation') are based solely on raw numerical comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw performance metrics are reported (Net Worth, Income, DAU). No standardized effect sizes, percentage improvements with baseline context, or magnitude-of-difference measures are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 5 runs for Vending and only 1 run for Freelance/Operation. No power analysis or discussion of whether the number of trials is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Table 2) for Freelance and Operation are explicitly single-run with no variance. Vending is averaged over 5 runs but Table 2 shows no standard deviation. Variance is shown only in appendix figures (Figures 3, 16-18) and one secondary experiment (Table 4 with mean ± std)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Eleven LLMs are compared against each other across all three environments (Table 2), and a human expert baseline is included for the Operation environment."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models include GPT-5.2 (2025-12-11), Gemini-3-Pro, Claude-Sonnet-4.5 (2025-09-29), DeepSeek-v3.2, and other recent models with specific version dates from 2025 (Appendix A, Tables 5-6)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple diagnostic studies: memory module ablation (Table 3, four memory types), context window length sweep (Figure 4, k=32 to 1024), thinking mode on/off (Figure 6), and environment complexity tiers (Table 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three distinct primary metrics across environments: Net Worth (Vending), Income (Freelance), and DAU (Operation). Additional behavioral pattern analysis and failure mode analysis supplement the quantitative metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human experts were recruited to perform the Operation task via a dedicated GUI (Appendix H), achieving average DAU of 1,404 over ~45 minutes, serving as a human performance baseline."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark environments are novel evaluation settings. No models were trained or tuned on these environments, so all results are on unseen test conditions with no dev/test contamination."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported per-environment (Table 2), per-model behavioral patterns over time (Figure 5, Appendix F), per-complexity tier (Table 4), and per-memory-type (Table 3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 'Failure Modes Analysis' provides detailed comparison of top-2 models per scenario, identifying specific failure modes: passive waiting (Gemini-3-Flash in Vending), redundant loops (Gemini-3-Pro in Freelance), and strategic misalignment (runner-up in Operation)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Three models score 0 income in Freelance (Table 2: DeepSeek-v3.2, Grok-4.1-Fast, Kimi-k2). Context window expansion degrades Gemini-3-Pro (Figure 4). Working memory hurts Gemini-3-Pro in Freelance. Gemini-3-Pro stagnates at higher complexity (Table 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claim that 'no single model dominates across all three scenarios' is supported by Table 2 showing different leaders per environment (Gemini-3-Pro in Vending, GPT-5-Mini in Freelance, Claude-Sonnet-4.5 in Operation). The 'significant suboptimality' claim is supported by failure mode analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims from ablation-style experiments use controlled single-variable manipulation: thinking on/off (Figure 6), memory module addition (Table 3), context window length variation (Figure 4). Each varies one factor while holding others constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Interactive Economies' broadly but tests only 3 specific simulated environments. The conclusion generalizes to 'frontier models struggle to maintain strategic coherence over long-time decisions' from 3 simulated games. The abstract claims a 'generalizable benchmark' without bounding the generalization."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations discussed for observed performance differences. The paper does not consider API configuration effects, model training data composition, whether benchmark design may favor certain architectures, or whether the stochastic environments introduce systematic biases."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Net Worth, Income, and DAU in simulated games and frames this as evaluating 'long-horizon plan-and-execute' capability in 'realistic economic settings.' The gap between simulated economic game metrics and actual economic decision-making capability is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A Tables 5-6 provide specific version identifiers with dates: 'GPT-5.2-2025-12-11', 'GPT-5-Mini-2025-08-07', 'Claude-Sonnet-4-5-20250929', 'Kimi-K2-0905-Preview', plus model repository links for open-weights models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system prompts provided in Appendix C: shared core protocol (C.1), Vending agent + market physics synthesis (C.2), Freelance agent + LLM auditor (C.3), and Operation agent (C.4)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'Generation parameters are standardized (Temperature=1.0, Top-p=0.95) across all trials.' Context window restricted to 128 steps. Maximum 365 days."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Agent scaffolding described: structured tool-calling mechanism (Appendix C.1), sliding context window of 128 steps, per-environment action schemas (Appendix D, Tables 7-9), and optional memory module architectures with formal definitions (Appendix E)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Freelance data pipeline described in Section 3.2.2: dataset aggregation → difficulty filtering → strategy router & mutation (scenario injection for code, logic mutation for quantitative) → solvability check via LLM-as-Judge. Vending data synthesis via Perplexity queries + LLM-generated market physics (Appendix C.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The conclusion mentions model capabilities generally but does not substantively discuss limitations of the benchmark design, evaluation methodology, or scope of the findings."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. No analysis of how simulated environments might not reflect real economic dynamics, no discussion of single-run reliability for Freelance/Operation, and no consideration of benchmark design biases."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper does not specify what the results do NOT show, what settings or populations are excluded, or what claims the authors are not making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (agent trajectories, daily metric logs, run outputs) is made available. Only aggregated results in tables and figures are provided. The GitHub repo contains benchmark code but not experimental outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection described per environment: Vending uses Perplexity queries with LLM-synthesized market physics (Section 3.2.1), Freelance aggregates from 8 public datasets with mutation pipeline (Section 3.2.2), Operation uses parametric continuous state space (Section 3.2.3)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For human baseline testing, the paper states only 'we recruited human experts' with no details about recruitment channels, number of participants, qualifications, or selection criteria."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The Freelance data pipeline is well-documented with explicit stages and filtering (Section 3.2.2). The Vending market physics synthesis pipeline is provided with the full prompt (Appendix C.2). Environment state transitions are formally specified (Appendix B)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed anywhere in the paper. Authors are from OPPO AI Agent Team, suggesting corporate funding, but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are identified as 'OPPO AI Agent Team' on the title page. Correspondence emails use @oppo.com domain."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OPPO is a technology company with commercial interests in AI agent capabilities. The benchmark evaluates LLM planning abilities relevant to OPPO's AI agent products. No discussion of funder independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, no patent disclosures, and no financial interest declarations are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the 11 evaluated models. Appendix A lists API version dates (e.g., GPT-5.2-2025-12-11) but these are release/snapshot dates, not training cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No systematic discussion of train/test overlap. The Freelance tasks are derived from widely-used public datasets (SWE-bench, GSM8K, LiveCodeBench) that are likely in model training data. While mutation is applied, overlap is not explicitly discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Contamination prevention is partially implemented for Freelance tasks via 'Logic Mutation, refactoring numerical values and variables to prevent data contamination or memorization' (Section 3.2.2), but no systematic contamination analysis is performed for the benchmark as a whole. Vending and Operation contamination risks are not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration mentioned for the human baseline evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval mentioned for the human expert recruitment and evaluation."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No demographics reported for human experts. Only described as 'human experts' with no characterization of number of participants, expertise domain, experience level, or background."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria stated for human expert recruitment."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluation is a performance measurement task, not a randomized experiment with treatment and control conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the human evaluation is a performance baseline task, not a comparative experiment requiring blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information provided. The paper notes 'maintaining consistent human attention proved to be a significant challenge' but does not report how many experts started vs. completed the evaluation or any dropout information."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token consumption, or per-evaluation costs are reported despite evaluating 11 models across 3 environments with 365-step horizons and multiple runs for some conditions."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated. No mention of total API spend, GPU hours, or hardware used for the evaluation campaign."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Five independent trials conducted for a representative subset of models across all three scenarios. Stochastic stability analysis shown in Figures 3, 16-18, revealing Vending has high variance while Freelance and Operation are more stable."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2: 'we conducted five independent trials for a representative subset of models across all three scenarios.' Main results report 5-run averages for Vending and single runs for Freelance and Operation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Fixed generation parameters (Temperature=1.0, Top-p=0.95) used without justification for these specific values. No exploration of alternative settings or discussion of why these were chosen."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Standardized generation parameters (Temperature=1.0, Top-p=0.95) used identically across all models and trials, avoiding cherry-picking. The context window default of 128 is explored via ablation (Figure 4)."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical hypothesis tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment of benchmark designer bias. OPPO designed EcoGym and evaluates all 11 models on it. The potential for benchmark design choices to systematically favor or disfavor certain model capabilities is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of compute budget vs performance. Models with vastly different inference costs (e.g., Qwen3-235B-A22B vs GPT-5-Mini) are compared on raw performance without controlling for or reporting compute differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the three simulated environments actually measure 'long-horizon planning capability' as claimed. No construct validity analysis, no comparison with alternative measures of planning ability, and no questioning of whether simulated economic games proxy real economic decision-making."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models use the same unified interface with standardized action schemas (Tables 7-9), identical context window settings (128 steps), the same tool-calling protocol (Appendix C.1), and the same generation parameters, controlling for scaffolding confounds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The Freelance tasks derive from datasets (LiveCodeBench, SWE-bench, GSM8K) that predate the evaluated models, and the temporal relationship between source data and model training is not analyzed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through prompts, action schemas, or feedback mechanisms that could advantage models familiar with similar patterns."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of non-independence. The underlying datasets (SWE-bench, GSM8K, FinQA, etc.) used for Freelance tasks are widely used in LLM training and evaluation, and potential overlap is not addressed beyond the mutation step."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.2 describes concrete prevention: 'Logic Mutation, refactoring numerical values and variables to prevent data contamination or memorization' for quantitative tasks, plus 'Scenario Injection' for coding tasks and a 'solvability check via an LLM-as-a-Judge' before task inclusion."
       }
     }
   },
   "claims": [
     {
       "claim": "No single model consistently achieves superior performance across all three EcoGym scenarios.",
       "evidence": "Table 2 shows different leaders: Gemini-3-Pro dominates Vending (11274.73), GPT-5-Mini leads Freelance (2990.72), and Claude-Sonnet-4.5 tops Operation (1572.49).",
       "supported": "strong"
     },
     {
       "claim": "Models exhibit significant suboptimality in either high-level strategies or efficient action execution.",
       "evidence": "Failure mode analysis (Section 4.2): Claude-Sonnet-4.5 won Operation via strategic prioritization (quantity over quality), while GPT-5-Mini won Freelance via execution efficiency (precise state tracking). Gemini-3-Pro suffered redundant loops in Freelance.",
       "supported": "moderate"
     },
     {
       "claim": "Expanding the context window does not yield consistent performance gains.",
       "evidence": "Figure 4 shows Gemini-3-Pro peaks at k=128 and degrades progressively to k=1024, while Gemini-3-Flash shows volatile performance across window sizes. Only tested on Operation environment with 2 models.",
       "supported": "moderate"
     },
     {
       "claim": "Memory integration generally enhances performance but is not universally beneficial, and optimal memory type is model- and task-dependent.",
       "evidence": "Table 3 shows all four memory types improve both models in Vending, but the paper notes performance regression for Gemini-3-Pro with working memory in Freelance. Gemini-3-Flash favors working memory while Gemini-3-Pro benefits from episodic memory.",
       "supported": "moderate"
     },
     {
       "claim": "Thinking mode catalyzes universal performance elevation across model variants.",
       "evidence": "Figure 6 shows DAU improvement from 1196.71 to 1398.20 for Gemini-3-Flash and from 1280.75 to 1511.08 for Gemini-3-Pro. Tested only on 2 models in 1 environment (Operation).",
       "supported": "weak"
     },
     {
       "claim": "Current SOTA LLMs have achieved super-human performance in specific long-horizon economic planning scenarios.",
       "evidence": "Human experts averaged DAU 1,404 in Operation; Claude-Sonnet-4.5 (1572.49) and Grok-4.1-Fast (1372.77) surpassed or approached this. Only tested in Operation with vaguely described human baseline (unspecified number of experts, no demographics).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-run main results for 2/3 environments",
       "detail": "Freelance and Operation main results (Table 2) are from single runs despite the paper acknowledging 'inherent instability' in long-horizon environments. Variance analysis in appendix is only for a subset of models."
     },
     {
       "flag": "Vague human baseline with super-human claims",
       "detail": "Human expert comparison lacks critical details: number of participants, expertise qualifications, recruitment method, and practice time are all unreported. A single average DAU number (1,404) from uncharacterized 'human experts' supports the claim of 'super-human performance.'"
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "All claims of model superiority are based on raw number comparisons without any significance testing, despite the paper demonstrating high variance in at least one environment (Vending)."
     },
     {
       "flag": "Conflict of interest: company-designed benchmark",
       "detail": "OPPO AI Agent Team designed EcoGym, selected the evaluation environments, and evaluates all models. No competing interests statement is provided and no benchmark designer bias is acknowledged."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or scope boundaries section. Simulated economic environments are presented as 'realistic economic settings' without discussing the gap between simulation and real-world economic dynamics."
     },
     {
       "flag": "No cost reporting for large-scale evaluation",
       "detail": "Evaluating 11 models across 3 environments with 365-day horizons, multiple ablations, and multiple runs requires substantial API costs, but no cost data is reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Vending-bench: A benchmark for long-term coherence of autonomous agents",
       "authors": ["Axel Backlund", "Lukas Petersson"],
       "year": 2025,
       "arxiv_id": "2502.15840",
       "relevance": "Direct predecessor benchmark for long-term agent coherence evaluation in economic settings; EcoGym builds upon its methodology."
     },
     {
       "title": "HeroBench: A benchmark for long-horizon planning and structured reasoning in virtual worlds",
       "authors": ["Petr Anokhin", "Roman Khalikov", "Stefan Rebrikov"],
       "year": 2025,
       "arxiv_id": "2508.12782",
       "relevance": "Benchmark for long-horizon planning with competitive resource management dynamics."
     },
     {
       "title": "GDPval: Evaluating AI model performance on real-world economically valuable tasks",
       "authors": ["Tejal Patwardhan", "Rachel Dias", "Elizabeth Proehl"],
       "year": 2025,
       "relevance": "Evaluates AI agents on macroeconomically meaningful real-world tasks spanning major economic sectors."
     },
     {
       "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts",
       "authors": ["Hjalmar Wijk", "Tao Lin", "Joel Becker"],
       "year": 2024,
       "arxiv_id": "2411.15114",
       "relevance": "Evaluates frontier AI capabilities on expert-level tasks with human comparison, relevant to agent capability assessment."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major LLM code generation benchmark; used as task source for EcoGym's Freelance environment."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code evaluation benchmark; used as task source for EcoGym's Freelance environment."
     },
     {
       "title": "WebArena: A realistic web environment for building autonomous agents",
       "authors": ["Shuyan Zhou", "Frank F. Xu", "Hao Zhu"],
       "year": 2024,
       "arxiv_id": "2307.13854",
       "relevance": "Realistic web agent evaluation environment relevant to agentic AI capability assessment."
     },
     {
       "title": "Generative agents: Interactive simulacra of human behavior",
       "authors": ["Joon Sung Park", "Joseph O'Brien", "Carrie Jun Cai"],
       "year": 2023,
       "relevance": "Foundational work on LLM-based agents with persistent memory and planning in social and economic interactions."
     },
     {
       "title": "AgentBoard: An analytical evaluation board of multi-turn LLM agents",
       "authors": ["Chang Ma", "Junlei Zhang", "Zhihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.13178",
       "relevance": "Multi-turn LLM agent evaluation framework with fine-grained analytical capabilities."
     },
     {
       "title": "The OpenHands software agent SDK: A composable and extensible foundation for production agents",
       "authors": ["Xingyao Wang", "Simon Rosenberg", "Juan Michelini"],
       "year": 2025,
       "arxiv_id": "2511.03690",
       "relevance": "Agent scaffold framework for production AI agents, relevant to agentic workflow evaluation."
     },
     {
       "title": "xBench: Tracking agents productivity scaling with profession-aligned real-world evaluations",
       "authors": ["Kaiyuan Chen", "Yixin Ren", "Yang Liu"],
       "year": 2025,
       "arxiv_id": "2506.13651",
       "relevance": "Benchmarks AI agent productivity scaling across professional domains."
     },
     {
       "title": "Remote Labor Index: Measuring AI automation of remote work",
       "authors": ["Mantas Mazeika", "Alice Gatti", "Cristina Menghini"],
       "year": 2025,
       "relevance": "Measures AI potential to automate remote labor, relevant to economic impact evaluation of AI agents."
     }
   ]
 }