ai-research-survey

scan.json (28194B)

---

 {
   "paper": {
     "title": "EComStage: Stage-wise and Orientation-specific Benchmarking for Large Language Models in E-commerce",
     "authors": [
       "Kaiyan Zhao",
       "Zijie Meng",
       "Zheyong Xie",
       "Jin Duan",
       "Yao Hu",
       "Zuozhu Liu",
       "Shaosheng Cao"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.02752",
     "doi": "10.48550/arXiv.2601.02752"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "EComStage benchmarks 33 LLMs across 7 e-commerce tasks decomposed into Perception, Planning, and Action stages, with both customer- and merchant-oriented scenarios. No single model excels across all stages and orientations: Claude Sonnet 4 leads on merchant-oriented planning tasks while Qwen3-235B-A22B-Instruct achieves the best overall score (85.61) among large open-source models. Most models perform well on classification-style Perception tasks but show significant variation on Planning and Action, with merchant-oriented tasks revealing particular weaknesses in models like GPT-4o.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository link in footnote 1: 'Codes and data: https://github.com/KYuuto1006/EComStage'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The same footnote 1 states 'Codes and data' are released at the GitHub link, and Table 1 lists Availability as checkmark for EComStage."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '8 NVIDIA H800 GPUs' and inference hyperparameters, but provides no requirements.txt, Dockerfile, or library version specifications needed to recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README descriptions, or scripts for replicating experiments are mentioned in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 reports only point estimates (e.g., '84.21', '85.61') with no confidence intervals, error bars, or ± notation for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'Claude Sonnet 4 achieves the best average score') based solely on comparing raw numbers. No statistical significance tests are used anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw accuracy scores and cosine similarity values are reported. No effect sizes (Cohen's d, odds ratios, or contextualized improvement magnitudes) are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes 'our Planning set contains only 164 samples' but provides no justification or power analysis for sample sizes across any of the 7 tasks."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1.3 explicitly states 'All experiments are conducted on 8 NVIDIA H800 GPUs with a single run.' No variance, standard deviation, or multiple-run results are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 33 models against each other, providing extensive cross-model comparisons across all 7 tasks (Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Evaluated models include very recent systems: Claude Sonnet 4, Gemini 2.5-Pro, Qwen3 series, DeepSeek-R1, and dots.llm1.inst, all from 2024-2025."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper evaluating existing models, not a system with components to ablate. The stage-wise breakdown serves a similar diagnostic purpose."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses accuracy for close-ended tasks and cosine similarity (using Qwen3-Embedding-8B) for open-ended generation tasks (Section 4.1.2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation of model outputs is entirely automated (accuracy and cosine similarity). No human evaluation of the models' generated responses is performed. Human annotation was used only for benchmark construction, not for evaluating model outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is used as a fixed evaluation set with uniform hyperparameters (temperature 0.1, top-p 0.001) applied to all models. No tuning is done on the benchmark data, so it functions as a clean held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-task breakdowns for all 7 tasks across all 33 models, and Figure 3 provides stage-wise and orientation-wise aggregations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses which models score low on which tasks in aggregate (e.g., 'GLM4-9B' scoring 38.44 on Attitude Classification) but provides no qualitative error analysis or concrete examples of model failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: DeepSeek-R1 performing worse than DeepSeek-V3 despite being newer (Section 4.2.4), Qwen3-8B and Qwen3-14B falling behind smaller Qwen3-4B-Instruct (Section 4.2.3), and GPT-4o showing weakness on merchant-oriented tasks (Section 4.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims stage/orientation-specific strengths and weaknesses, which is supported by Table 3 and Figure 3. The claim that 'no single model consistently excels across all stages' is directly supported by the per-task results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes multiple unsupported causal claims: 'likely due to its optimization for complex reasoning and tool use' (Section 4.2.1), 'likely benefiting from its large model capacity' (Section 4.2.4), 'likely due to its training on fine-grained dialogue understanding' (Section 4.2.4). These are speculative explanations without causal study design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to benchmark LLMs 'in E-commerce' generally, but the data comes from a single platform (Xiaohongshu, as indicated by author affiliations). The paper does not acknowledge that results from one e-commerce platform's data may not generalize to other platforms, markets, or cultural contexts."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes performance differences to model-specific factors (training data, instruction tuning) without considering alternative explanations such as prompt sensitivity, language bias from translation, or task-specific confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures accuracy on simplified tasks (e.g., classification, matching) and frames this as evaluating 'Perception, Planning, and Action' capabilities of e-commerce agents. The gap between controlled benchmark performance and real-world agent capability in live e-commerce settings is not discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Closed-source APIs are listed as 'GPT-4o', 'Gemini 2.5-Pro', 'Claude 3.7', 'Claude Sonnet 4' without snapshot dates or API version identifiers. The schema requires specific versions, not marketing names."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system prompts and user prompt templates for all 7 tasks are provided in Appendix A.2 (Figures 9-15), with sufficient detail to reconstruct the inputs sent to models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 reports: batch size 32, input length 4096 tokens, temperature 0.1, top-p 0.001, repetition penalty 1.05, max generation 512 tokens."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are evaluated via direct prompting without tools, memory, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes a detailed multi-stage pipeline including task-specific filtering (Figure 5), global filtering (Figure 6), consistency checking (Figure 7), translation (Figure 8), and security screening, with specific criteria at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, discussing the lack of error propagation evaluation and limited coverage of e-commerce domains."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations mention specific threats: the benchmark 'does not capture error propagation across stages, which may occur in real-world deployments' and covers 'representative but finite e-commerce scenarios' that 'do not fully encompass all e-commerce domains.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what the benchmark does not cover: cross-stage error propagation and domains beyond the seven covered tasks, noting room for future expansion."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states data is released at the GitHub repository (footnote 1), and Table 1 marks Availability with a checkmark."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.1 describes data collection from 'real e-commerce service scenarios, covering both customer and merchant orientations.' The pipeline including annotation, filtering, and verification is described in detail."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Annotators are described as 'employees of our company with relevant experience in e-commerce operations' (Appendix A.3), but the sampling strategy for selecting which business data to include from the platform is not described — there is no information on how scenarios were selected from the universe of available interactions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline stages are described (Figure 2, Section 3.1), but no intermediate counts are provided. The paper does not report how many samples existed before filtering, how many were removed at each stage, or the rejection rates for quality checks."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is disclosed anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: The University of Tokyo, Zhejiang University, and Xiaohongshu Inc. The corresponding author's email is at xiaohongshu.com."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three authors are from Xiaohongshu Inc., which is the apparent source of the benchmark data. The company has an interest in their platform's data being seen as a valuable source for AI evaluation, though they are not evaluating their own model."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declarations are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 33 evaluated models, including closed-source APIs like GPT-4o and Gemini 2.5-Pro."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any model's training data might contain similar e-commerce conversations or the specific tasks used in the benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Although the benchmark data is newly created from proprietary business data (reducing contamination risk), the paper does not discuss contamination at all. The data was translated from Chinese to English, which could mitigate overlap, but this is not framed as a contamination prevention measure."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Annotators are company employees performing work duties, not research subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, latency, or per-example costs are reported despite evaluating 33 models (including paid APIs) across 4,804 samples."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '8 NVIDIA H800 GPUs' but does not quantify total GPU hours, wall-clock time, or API spend for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1.3 states 'All experiments are conducted on 8 NVIDIA H800 GPUs with a single run.' No seed sensitivity analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 explicitly states 'with a single run,' making the number of runs clear."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses fixed hyperparameters (temperature 0.1, top-p 0.001, repetition penalty 1.05) but does not justify these choices or report whether alternatives were explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper applies uniform settings across all models (Section 4.1.3: 'These settings are applied consistently across all evaluated models to provide a fair comparison'), so there is no cherry-picking of per-model configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 33 models across 7 tasks, making numerous comparative claims, but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates published third-party models on a benchmark; it does not propose or evaluate its own model, so the Lucic et al. bias of re-implementing baselines does not apply."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 1B to 200B+ parameters are compared without normalizing for compute. The paper does not discuss performance per unit of compute or compare models at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper defines Perception, Planning, and Action stages and assigns tasks to them, but does not validate that these tasks actually measure the claimed constructs. For example, whether 'Query Match' truly measures 'Perception' capability rather than simple retrieval/matching is not discussed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used; models are evaluated via direct prompting with uniform settings."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models' training data might include similar e-commerce interactions from before the benchmark's creation date."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (e.g., providing question lists or response options) gives models information that would not be available in a real deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data for any evaluated model might contain similar customer service conversations from the same or similar e-commerce platforms."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. Although the data originates from proprietary sources and is translated, this is not framed or analyzed as a contamination mitigation strategy."
       }
     }
   },
   "claims": [
     {
       "claim": "No single model consistently excels across all stages (Perception, Planning, Action) or orientations (customer-oriented, merchant-oriented).",
       "evidence": "Table 3 and Figure 3 show varying performance across models: Claude Sonnet 4 leads overall (84.21) but Gemini 2.5-Pro leads on Solution Decision (87.06); GPT-4o shows relative weakness on merchant-oriented tasks while DeepSeek-V3 excels there.",
       "supported": "strong"
     },
     {
       "claim": "Qwen3-235B-A22B-Instruct achieves the best overall performance (85.61) among all evaluated models.",
       "evidence": "Table 3 shows Qwen3-235B-A22B-Instruct at 85.61 average across 7 tasks, the highest value in the table.",
       "supported": "moderate"
     },
     {
       "claim": "Recent instruction-tuned variants consistently outperform their base counterparts.",
       "evidence": "Section 4.2.4: Qwen3-235B-A22B-Instruct (85.61) outperforms Qwen3-235B-A22B (84.43); Qwen3-30B-A3B-Instruct (83.63) outperforms Qwen3-30B-A3B (82.05). Section 4.2.2: Qwen3-4B-Instruct (82.26) substantially outperforms Qwen3-4B (78.13).",
       "supported": "moderate"
     },
     {
       "claim": "Stage-wise evaluation reveals weaknesses hidden by overall average scores.",
       "evidence": "Figure 3 shows models with similar overall averages have distinct stage-specific profiles. GPT-4o performs well overall but is weaker on merchant-oriented tasks; Claude Sonnet 4 excels in Planning but lags in Action.",
       "supported": "moderate"
     },
     {
       "claim": "Smaller models (4B) can achieve performance competitive with much larger models when properly instruction-tuned.",
       "evidence": "Table 3 shows Qwen3-4B-Instruct (82.26) outperforming Qwen3-8B (81.01) and Qwen3-14B (81.70), though this is based on single-run results without variance estimates.",
       "supported": "weak"
     },
     {
       "claim": "DeepSeek-V3 performs better than DeepSeek-R1 in e-commerce scenarios.",
       "evidence": "Table 3: DeepSeek-V3 (84.23) vs DeepSeek-R1 (82.17). The difference is 2.06 points but based on a single run with no statistical test.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-run evaluation with no variance estimates",
       "detail": "All 33 models are evaluated in a single run with no repetition. Performance differences of 1-2 points between models (e.g., DeepSeek-V3 vs DeepSeek-R1) could easily fall within run-to-run variance, yet are interpreted as meaningful."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper makes dozens of comparative claims about model superiority based purely on raw accuracy differences, with no statistical significance tests. Many claimed differences are small (e.g., 84.21 vs 84.01 between Claude Sonnet 4 and Gemini 2.5-Pro)."
     },
     {
       "flag": "Company employees creating benchmark from own platform data",
       "detail": "Three authors from Xiaohongshu Inc. created a benchmark using data from their own platform. The data source is never named in the paper despite being identifiable from affiliations. Potential bias in task selection, data curation, and what constitutes 'correct' answers."
     },
     {
       "flag": "No contamination analysis",
       "detail": "33 pre-trained LLMs are evaluated on e-commerce tasks without any discussion of whether similar customer service data appeared in their training sets. Many models are trained on web data that likely includes e-commerce conversations."
     },
     {
       "flag": "Cosine similarity as evaluation metric for generation tasks",
       "detail": "Open-ended generation tasks (Query Rewrite, RAG-QA) are evaluated via cosine similarity with a single embedding model (Qwen3-Embedding-8B), which may not capture semantic correctness. This is validated neither against human judgments nor with alternative embedding models."
     },
     {
       "flag": "Speculative causal attributions",
       "detail": "The paper repeatedly attributes performance differences to model-specific factors ('likely due to its optimization for complex reasoning,' 'likely benefiting from its large model capacity') without evidence for these causal claims."
     }
   ],
   "cited_papers": [
     {
       "title": "τ-bench: A benchmark for tool-agent-user interaction in real-world domains",
       "authors": ["Shunyu Yao", "Noah Shinn", "Pedram Razavi", "Karthik Narasimhan"],
       "year": 2024,
       "arxiv_id": "2406.12045",
       "relevance": "Key related benchmark for evaluating LLM agents in realistic tool-use scenarios including retail domains."
     },
     {
       "title": "AgentBench: Evaluating LLMs as agents",
       "authors": ["Xiao Liu"],
       "year": 2024,
       "relevance": "Benchmark for evaluating LLMs as agents across multiple environments, relevant to agentic AI evaluation methodology."
     },
     {
       "title": "Large language model based multi-agents: A survey of progress and challenges",
       "authors": ["Taicheng Guo"],
       "year": 2024,
       "arxiv_id": "2402.01680",
       "relevance": "Survey of multi-agent LLM systems covering coordination, reasoning, and evaluation challenges."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": ["Zhiheng Xi"],
       "year": 2023,
       "arxiv_id": "2309.07864",
       "relevance": "Comprehensive survey of LLM-based agents covering capabilities, architectures, and evaluation approaches."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["Aaron Grattafiori"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Technical report for the Llama 3 model family, one of the major open-source LLMs evaluated in benchmarks."
     },
     {
       "title": "DeepSeek-V3 technical report",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2412.19437",
       "relevance": "Technical report for DeepSeek-V3, a major open-source model relevant to LLM capability evaluation."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Describes reinforcement learning approach to improving LLM reasoning, relevant to agent capability research."
     },
     {
       "title": "GPT-4o system card",
       "authors": ["OpenAI"],
       "year": 2024,
       "arxiv_id": "2410.21276",
       "relevance": "System card for GPT-4o, a key closed-source model evaluated across benchmarks."
     },
     {
       "title": "WebShop: Towards scalable real-world web interaction with grounded language agents",
       "authors": ["Shunyu Yao", "Howard Chen", "John Yang", "Karthik Narasimhan"],
       "year": 2022,
       "relevance": "Early benchmark for evaluating LLM agents in web-based shopping environments."
     },
     {
       "title": "Large language model agent: A survey on methodology, applications and challenges",
       "authors": ["Junyu Luo"],
       "year": 2025,
       "arxiv_id": "2503.21460",
       "relevance": "Recent comprehensive survey of LLM agent methodology and evaluation, relevant to understanding the agentic AI landscape."
     },
     {
       "title": "A survey on code generation with LLM-based agents",
       "authors": ["Yihong Dong"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey of LLM-based code generation agents, relevant to understanding agent capabilities in software engineering."
     },
     {
       "title": "A survey of large language models",
       "authors": ["Wayne Xin Zhao"],
       "year": 2025,
       "arxiv_id": "2303.18223",
       "relevance": "Comprehensive survey of LLMs covering training, evaluation, and capabilities, relevant to overall LLM assessment methodology."
     }
   ]
 }