ai-research-survey

scan.json (28880B)

---

 {
   "paper": {
     "title": "Magentic Marketplace: An Open-Source Environment for Studying Agentic Markets",
     "authors": [
       "Gagan Bansal",
       "Wenyue Hua",
       "Zezhou Huang",
       "Adam Fourney",
       "Amanda Swearngin",
       "Will Epperson",
       "Tyler Payne",
       "Jake M. Hofman",
       "Brendan Lucier",
       "Chinmay Singh",
       "Markus Mobius",
       "Akshay Nambi",
       "Archana Yadav",
       "Kevin Gao",
       "David M. Rothschild",
       "Aleksandrs Slivkins",
       "Daniel G. Goldstein",
       "Hussein Mozannar",
       "Nicole Immorlica",
       "Maya Murad",
       "Matthew Vogel",
       "Subbarao Kambhampati",
       "Eric Horvitz",
       "Saleema Amershi"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.25779"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Available open-source at https://github.com/microsoft/multi-agent-marketplace' (Section 1, footnote 1) and Contribution 3 explicitly states they 'open-source Magentic Marketplace.'"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses fully synthetic data generated through a pipeline described in Section 4.1, but there is no explicit statement that the generated datasets (the 33-customer/99-business and 100-customer/300-business scenarios) are included in the repository. The paper says 'See our repository for additional synthetic domain data and experiments' but does not confirm whether the exact experimental datasets are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is mentioned in the paper. The paper mentions vLLM for open-source models but does not provide a comprehensive environment setup with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. While the methodology is explained at a high level, there are no specific commands, scripts, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results with error bars in Figures 4, 5, 7, 8, and 9. Section 4.2 states 'All experiments were conducted with 5 independent runs, with mean and standard deviation reported.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'GPT-4.1 and Gemini-2.5-Flash come very close to the optimal outcome') but does not report any statistical significance tests (no p-values, t-tests, or other formal comparisons)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports concrete effect sizes with context: 'consumer welfare declines by 4.3% when providing one hundred search results versus three' (Section 5.2), 'first proposals achieving selection rates between 60-100% compared to near-zero selection for third proposals. This represents a 10-30 fold advantage' (Section 5.4), and 'Sonnet-4 - 65.4%, GPT-5 - 44%' decline figures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 5 independent runs per condition and two market scales (33/99 and 100/300) but provides no justification for why these specific sizes were chosen, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 explicitly states 'All experiments were conducted with 5 independent runs, with mean and standard deviation reported.' Standard deviation is visible in the box plots in the figures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 defines four baselines: Random w/ items only, Cheapest w/ items & prices, Random w/ items & amenities, and Optimal. These are compared against the agentic conditions in Figure 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines represent different information conditions in the marketplace (random selection, price-optimized, amenity-aware, and theoretical optimal). These are appropriate structural baselines for a novel environment since there are no prior systems to compare against. The paper also compares multiple contemporary frontier models (GPT-4.1, GPT-5, Sonnet-4, Sonnet-4.5, Gemini-2.5-Flash)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The experimental design systematically varies components: search condition (perfect vs. lexical), consideration set size (3 to 100), and manipulation conditions (6 strategies). The welfare experiments with and without perfect search effectively ablate the discovery layer. The consideration set experiments ablate the impact of information overload."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: consumer welfare/utility (Section 5.1), mean payments to manipulated businesses (Section 5.3), position selection rates (Section 5.4), proposal selection rates (Section 5.4), and number of businesses contacted (Figure 6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Our manual evaluation of Qwen3-14b revealed significant performance limitations' and 'manual analysis of seven trials' (Section 5.1), but this is only an informal diagnostic for one model's failure modes, not a systematic human evaluation of the system's outputs or agent decisions."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a simulation-based study where agents operate in a synthetic marketplace environment. There is no train/test split paradigm; the evaluation is based on agent performance in the simulated market."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (GPT-4o, GPT-4.1, GPT-5, Gemini-2.5-Flash, etc.), per domain (Mexican restaurants, Contractors), per experimental condition (perfect search vs. lexical search), and per manipulation strategy (control, authority, social proof, loss aversion, prompt injection basic/strong)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses Qwen3-14b failure modes in detail: 'premature termination without completing payment, role confusion where it critiqued its own wrong actions while simultaneously executing them, and excessive purchasing without selection criteria.' Section 5.4 discusses the universal first-proposal bias as a systematic failure."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: the paradox of choice effect where more options reduced welfare (Section 5.2), severe first-proposal bias across all models (Section 5.4), manipulation vulnerability in smaller/older models (Section 5.3), and Qwen3-14b's fundamental inability to navigate the marketplace."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'frontier models can approach optimal welfare—but only under ideal search conditions' (supported by Figure 4), 'Performance degrades sharply with scale' (supported by consideration set experiments in Figure 5), and 'all models exhibit severe first-proposal bias, creating 10-30x advantages for response speed over quality' (supported by Figure 9 in Section 5.4). All claims are substantiated in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about the effect of consideration set size on welfare and of manipulation strategies on agent behavior. These are supported by controlled experiments where individual variables are manipulated while others are held constant. The experimental design (Table 2) systematically isolates factors. The ablation-style comparison between perfect search and lexical search is a controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "While the paper notes that it uses fully synthetic data and focuses on restaurants and contractors domains, the title 'An Open-Source Environment for Studying Agentic Markets' and broad claims like 'These findings reveal how behaviors emerge across market conditions, informing the design of fair and efficient agentic marketplaces' extend beyond what was tested. The paper does not systematically bound its claims to the tested setting (synthetic data, two specific domains, static markets)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses alternative explanations for the paradox of choice: 'We hypothesize that this effect may not be inherent to agentic markets; rather, it could arise from an interplay between limitations of agents and our specific design decisions.' Section 5.1 discusses prompt-model misalignment as an alternative explanation for Qwen3-14b failures but then provides evidence for more fundamental issues."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names without specific version identifiers: 'GPT-4o', 'GPT-4.1', 'GPT-5', 'Gemini-2.5-Flash', 'Sonnet-4', 'Sonnet-4.5'. No API snapshot dates or specific model version strings (e.g., 'gpt-4o-2024-08-06') are provided. The citations point to model cards but the actual versions used in experiments are not specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes agent roles and behaviors at a high level but does not provide the actual system prompts or instructions given to the LLM agents. Section 4.2 mentions agents are 'instructed to maximize utility' but the actual prompt text is not provided in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other hyperparameter settings are reported for any of the models used. The paper mentions using vLLM for open-source models and YARN for context extension but does not specify sampling parameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides detailed descriptions of the agentic scaffolding: the three-endpoint architecture (register, protocol, action), the five core actions (search, send text, send order proposals, send payments, receive), the action-observation loop, and the REST API specifications in Table 1. Figure 2 and Figure 3 show detailed architecture diagrams. The ReACT-style agent loop is mentioned in Section 5.1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the three-stage synthetic data generation pipeline in detail: (1) Item/Service Universe Construction with price distributions, (2) Customer Synthesis with item sampling and constraints, (3) Business Synthesis with controlled matching properties. The generation process is fully documented with parameter choices (e.g., 1-3 items, 1-2 amenities, α=2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Discussion section (Section 6) mentions some limitations in passing (e.g., 'our experiments focused on static markets') but there is no substantive dedicated section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are systematically discussed. The paper does not address issues such as the generalizability of synthetic data, the limited number of runs (5), the potential impact of prompt design on results, or the representativeness of the restaurant/contractor domains."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Discussion mentions 'our experiments focused on static markets' and notes future extensions to dynamic markets and human-in-the-loop designs, the paper does not explicitly state what the results do NOT show. There are no systematic scope boundaries like METR's Table 2 ('What the evidence does not show')."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (agent conversation logs, transaction records, per-trial results) is made available. Only aggregated results are shown in figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the synthetic data generation pipeline in detail, and Section 4.2 describes the experimental protocol including how agents interact with the marketplace, the utility function (Equation 1), and the evaluation conditions (Table 2)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study uses synthetic customer and business data with LLM agents. Data source is a fully synthetic generation pipeline, not a standard benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data generation pipeline is documented in Section 4.1 with three explicit stages and parameters. The experimental pipeline (how agents interact, make decisions, and complete transactions) is documented in Sections 3 and 4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgment of financial support is present. The Acknowledgements section (Section 8) thanks individuals for conversations and feedback but does not mention funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Microsoft for most authors, Arizona State University for Subbarao Kambhampati. The roles are also specified (Core Contributors, Contributors, Technical Program Managers, Advisors, Principal Investigator)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The paper is authored primarily by Microsoft employees and evaluates models that include Microsoft's own GPT-family models (GPT-4o, GPT-4.1, GPT-5, GPT-OSS-20b) via their API. Microsoft has a direct financial interest in the perception of these models' capabilities. No acknowledgment of this conflict is made."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Given that the authors are Microsoft employees evaluating Microsoft-affiliated models and proposing an environment that could influence how agentic markets are built, a financial interests declaration would be expected."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capabilities on a benchmark. It evaluates agent behavior in a novel simulated marketplace environment with fully synthetic data. The agents' performance depends on reasoning and tool-use capabilities, not on memorized benchmark answers."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The marketplace environment is novel and the data is fully synthetic, generated for this study. There is no risk of train/test overlap in the traditional sense."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The evaluation uses a novel simulated environment with synthetic data, not an established benchmark that could have appeared in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study. All agents are LLM-based and interact with synthetic data."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study uses fully synthetic data and LLM agents."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported. The paper runs experiments with multiple frontier models (including GPT-5, Sonnet-4.5) across many conditions with 5 runs each, but the total cost is not disclosed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, API spend, or hardware specifications are provided. The paper mentions using vLLM for open-source models but does not specify what hardware was used or the total compute requirements."
       }
     }
   },
   "claims": [
     {
       "claim": "Frontier models can approach optimal welfare outcomes under ideal (perfect) search conditions in two-sided agentic markets.",
       "evidence": "Figure 4 shows GPT-4.1 and Gemini-2.5-Flash approaching the optimal dashed line under the Agentic: Perfect search condition in both Mexican restaurant and Contractor domains (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Performance degrades sharply with increased consideration set size, exhibiting a 'paradox of choice' effect.",
       "evidence": "Figure 5 shows consumer welfare declining as consideration set size increases from 3 to 100. GPT-4o shows 4.3% decline; Sonnet-4 shows 65.4% decline; GPT-5 shows 44% decline on Mexican 100-300 (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "All models exhibit severe first-proposal bias, creating 10-30x advantages for response speed over quality.",
       "evidence": "Figure 9 shows first-proposal selection rates between 60-100% across all models, with near-zero rates for third proposals. GPT-4o and Sonnet-4.5 showed 100% first-proposal selection in some conditions (Section 5.4).",
       "supported": "strong"
     },
     {
       "claim": "Frontier models demonstrate robust manipulation resistance while smaller/older models are significantly vulnerable.",
       "evidence": "Figure 7 shows GPT-4.1, Sonnet-4.5, and Gemini-2.5-Flash maintaining stable performance across manipulation conditions, while GPT-4o, GPT-OSS-20B, and Qwen3-4B-2507 showed significant vulnerability (Section 5.3).",
       "supported": "strong"
     },
     {
       "claim": "Two-sided agentic markets can achieve reasonable welfare outcomes by reducing information asymmetries through agent-mediated communication.",
       "evidence": "Section 5.1 shows that under lexical search, proprietary models outperform two of three baselines (random selection and cheapest-by-price), demonstrating agents can navigate noisy discovery and make quality decisions.",
       "supported": "moderate"
     },
     {
       "claim": "Open-source models (GPT-OSS-20b, Qwen3-4b) can approach proprietary model performance under perfect search but degrade notably under lexical search.",
       "evidence": "Section 5.1 and Figure 4 show GPT-OSS-20b outperforming GPT-4o under both search conditions in the Mexican dataset, but notes 'an overall notable drop under lexical search' for open-source models generally.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "Magentic Marketplace is an open-source simulated environment for studying LLM agent behavior in two-sided economic markets. Experiments across multiple frontier and open-source models reveal that agents can approach optimal welfare under ideal search conditions, but performance degrades with larger consideration sets (a 'paradox of choice'). All tested models exhibit severe first-proposal bias (60-100% selection of first offers), creating 10-30x advantages for response speed over quality. Frontier models show robust manipulation resistance, but smaller models remain vulnerable to both psychological tactics and prompt injection attacks.",
   "red_flags": [
     {
       "flag": "Company evaluating its own models",
       "detail": "Microsoft employees evaluate GPT-family models (GPT-4o, GPT-4.1, GPT-5, GPT-OSS-20b) which are Microsoft/OpenAI products, alongside competitors. No conflict of interest statement is provided. The environment design itself could favor certain model architectures."
     },
     {
       "flag": "No significance testing",
       "detail": "Despite comparative claims across models and conditions, no statistical significance tests are reported. With only 5 runs per condition, the observed differences may not be statistically significant, particularly for close comparisons."
     },
     {
       "flag": "Missing model versions and hyperparameters",
       "detail": "No specific model version strings (API snapshots) or sampling hyperparameters (temperature, top-p) are reported. Different temperature settings could substantially affect agent behavior, making results difficult to reproduce."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section despite making broad claims about agentic market design. Key limitations such as synthetic-only data, limited domains, static markets, and prompt sensitivity are not systematically acknowledged."
     },
     {
       "flag": "Missing prompts",
       "detail": "The actual system prompts given to LLM agents are not provided, despite being critical to agent behavior. The paper describes agent roles at a high level but readers cannot reproduce the exact experimental setup without knowing the prompt text."
     }
   ],
   "cited_papers": [
     {
       "title": "Cooperation, competition, and maliciousness: LLM-stakeholders interactive negotiation",
       "authors": ["Sahar Abdelnabi", "Amr Gomaa", "Sarath Sivaprasad", "Lea Schönherr", "Mario Fritz"],
       "year": 2024,
       "relevance": "Studies LLM agent behavior in negotiation games, directly relevant to understanding agent capabilities in economic interactions."
     },
     {
       "title": "Multi-agent risks from advanced AI",
       "authors": ["Lewis Hammond", "Alan Chan", "Jesse Clifton"],
       "year": 2025,
       "arxiv_id": "2502.14143",
       "relevance": "Surveys risks from multi-agent AI systems, relevant to safety and alignment concerns in agentic markets."
     },
     {
       "title": "Generative agents: Interactive simulacra of human behavior",
       "authors": ["Joon Sung Park", "Joseph C. O'Brien", "Carrie J. Cai", "Meredith Ringel Morris", "Percy Liang", "Michael S. Bernstein"],
       "year": 2023,
       "arxiv_id": "2304.03442",
       "relevance": "Foundational work on LLM-based agents simulating human behavior, relevant to understanding agent capabilities in social and economic settings."
     },
     {
       "title": "The agentic economy",
       "authors": ["David M. Rothschild", "Markus Mobius", "Jake M. Hofman"],
       "year": 2025,
       "arxiv_id": "2505.15799",
       "relevance": "Conceptual framework for agentic economies and two-sided agentic markets, directly motivates the Magentic Marketplace work."
     },
     {
       "title": "Algorithmic collusion by large language models",
       "authors": ["Yannai A. Gonczarowski", "Ran I. Shorrer", "Sara Fish"],
       "year": 2024,
       "relevance": "Studies LLM collusion in economic settings, relevant to understanding risks in agentic markets."
     },
     {
       "title": "What is your AI agent buying? Evaluation, implications, and emerging questions for agentic e-commerce",
       "authors": ["Amine Allouah", "Omar Besbes", "Josue D Figueroa", "Yash Kanoria", "Akshit Kumar"],
       "year": 2025,
       "arxiv_id": "2508.02630",
       "relevance": "Evaluates AI agent behavior in e-commerce, directly relevant to understanding agentic purchasing decisions."
     },
     {
       "title": "From LLM reasoning to autonomous AI agents: A comprehensive review",
       "authors": ["Mohamed Amine Ferrag", "Norbert Tihanyi", "Merouane Debbah"],
       "year": 2025,
       "arxiv_id": "2504.19678",
       "relevance": "Comprehensive survey of autonomous AI agents, relevant to understanding the landscape of agentic AI capabilities."
     },
     {
       "title": "A survey of AI agent protocols",
       "authors": ["Yingxuan Yang", "Huacan Chai", "Yuanyi Song"],
       "year": 2025,
       "arxiv_id": "2504.16736",
       "relevance": "Surveys communication protocols for AI agents, relevant to the protocol design aspects of agentic markets."
     },
     {
       "title": "CRMArena: Understanding the capacity of LLM agents to perform professional CRM tasks in realistic environments",
       "authors": ["Kung-Hsiang Huang", "Akshara Prabhakar", "Sidharth Dhawan"],
       "year": 2024,
       "arxiv_id": "2411.02305",
       "relevance": "Evaluates LLM agents in professional CRM tasks, relevant to understanding agent capability in business interactions."
     },
     {
       "title": "The AI economist: Improving equality and productivity with AI-driven tax policies",
       "authors": ["Stephan Zheng", "Alexander Trott", "Sunil Srinivasa", "Nikhil Naik"],
       "year": 2020,
       "arxiv_id": "2004.13332",
       "relevance": "Foundational work on AI agents in economic simulation environments, directly relevant to the economic simulation paradigm."
     },
     {
       "title": "Virtual agent economies",
       "authors": ["Nenad Tomasev", "Matija Franklin", "Joel Z. Leibo"],
       "year": 2025,
       "arxiv_id": "2509.10147",
       "relevance": "Studies virtual economies with AI agents, directly relevant to understanding market dynamics in agentic settings."
     },
     {
       "title": "STRIDE: A tool-assisted LLM agent framework for strategic and interactive decision-making",
       "authors": ["Chuanhao Li", "Runhan Yang", "Tiankai Li"],
       "year": 2024,
       "arxiv_id": "2405.16376",
       "relevance": "LLM agent framework for strategic decision-making, relevant to agent architecture for economic interactions."
     }
   ]
 }