ai-research-survey

scan.json (26244B)

---

 {
   "paper": {
     "title": "Beyond Quantity: Trajectory Diversity Scaling for Code Agents",
     "authors": [
       "Guhong Chen",
       "Chenghao Sun",
       "Cheng Fu",
       "Qiyao Wang",
       "Zhihong Huang",
       "Chaopeng Wei",
       "Guangxu Chen",
       "Feiteng Fang",
       "Ahmadreza Argha",
       "Bing Zhao",
       "Xander Xu",
       "Qi Han",
       "Hamid Alinejad-Rokny",
       "Qiang Qu",
       "Binhua Li",
       "Shiwen Ni",
       "Min Yang",
       "Hu Wei",
       "Yongbin Li",
       "Yu Ding"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.03219"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We plan to open-source all components that can be publicly released, subject to licensing, privacy, and internal policy constraints' (footnote 1). This is a promise of future release with caveats, not a currently available artifact. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The synthesized trajectory dataset is not released. The paper mentions a plan to release but provides no download link or repository. The benchmarks used (BFCL, tau2-Bench, etc.) are public, but the paper's own synthesized data is not available."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.2 specifies training setup: Megatron-LM, 8x80GB GPUs per node, BF16 precision, Flash Attention v2, Gradient Checkpointing, global batch size 16, learning rate 1e-5 with cosine decay, 50 warmup steps, sequence length 65,536 tokens. This is sufficient to recreate the training environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While hyperparameters and system prompts are provided in appendices, there are no step-by-step reproduction instructions, no README with commands, and no scripts to replicate experiments. The paper describes the framework conceptually but does not provide actionable instructions a researcher could follow."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates (e.g., '36.66%', '40.44%'). No confidence intervals, error bars, or ± notation are provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., TDScaling 'outperformed' baselines, 'surpassing' prior SOTA) but no statistical significance tests are reported. Comparisons are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report absolute improvements with baseline context, e.g., '+7.25' improvement from 29.41% to 36.66% on BFCL, '+4.00' overall from 30.99% to 34.99% on coding benchmarks. This provides enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 500 and 5,000 training samples is described but not justified with any formal analysis. No power analysis or rationale for why these specific sizes were chosen beyond 'maximum common data availability across open-source projects' for 5,000."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. All results appear to be single-run numbers. There is no mention of averaging over multiple seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 compare against multiple baselines: proprietary models (GPT-5, GPT-4.1, Claude-Sonnet-4, Gemini-2.5-pro), open-source models (DeepSeek-V3.2, Qwen3-Coder-480B), and tool-learning methods (APIGen-MT, TOUCAN, Simia)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models and methods: GPT-5, Claude-Sonnet-4, DeepSeek-V3.2, and recent tool-learning methods APIGen-MT (2025), TOUCAN (2025), Simia (2025). These are recent and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents a systematic ablation study removing individual components: w/o Cluster Sampling, w/o Global Evolution, w/o Code Tool, and w/o All. Each variant is evaluated across all benchmarks."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on multiple benchmarks across two dimensions: general tool-use (BFCL Multi-turn, tau2-Bench with three domains) and coding tasks (RebenchT with OH-p@1 and Qod-p@1, CodeCI avg@2, BIRD p@1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via benchmark test suites. Given claims about 'generalization' and 'real-world' applicability, human evaluation of trajectory quality or agent behavior would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established external benchmarks (BFCL, tau2-Bench, SWE-rebench, CodeCI/LiveCodeBench, BIRD) as test sets. These are separate from the training data, which consists of synthesized trajectories from MCP tool definitions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down tau2-Bench results by domain (AIR, RET, TEL). Table 2 breaks down coding results by benchmark and metric (RebenchT OH-p@1 and Qod-p@1, CodeCI, BIRD). Table 3 provides per-benchmark ablation results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases of baselines (negative transfer in APIGen-MT, Simia, TOUCAN on coding benchmarks). Figure 10 includes an annotated case study comparing baseline failures (misinterpreted errors, wrong parameters) with the model's correct behavior. Figure 4 discusses inverse scaling failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 ablation shows the w/o Code Tool variant scores higher on BFCL (37.56% vs. 36.66%) at the cost of coding performance, explicitly discussed as a trade-off. The paper also reports that baseline methods suffer negative transfer."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims TDScaling improves both tool-use generalization and coding proficiency, and that diversity scaling attains a higher performance ceiling than quantity scaling. Tables 1-2 support the first claim, and Figure 4 supports the second. The abstract is appropriately hedged."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (Table 3), where removing individual components (Cluster Sampling, Global Evolution, Code Tool) in a controlled manner shows their contribution. This is adequate causal design for component-level claims. The ablation is systematically structured."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Trajectory Diversity Scaling for Code Agents' and claims like 'establishing a resource-efficient paradigm for training robust code agents' are broad. Results are tested only on Qwen3-Coder-30B-A3B and Qwen3-30B-A3B with Qwen3-Max as teacher. Generalization to other model families, sizes, or teacher models is not tested but is implied by the broad framing."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why TDScaling outperforms baselines. For instance, the teacher model (Qwen3-Max) could be a confound since baselines may use different or weaker teachers. The quality filtering pipeline differences are not controlled for. No threats-to-validity section addresses these."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Qwen3-Coder-30B-A3B-Instruct', 'Qwen3-30B-A3B-Instruct', and 'Qwen3-Max' without version numbers or snapshot dates. Baseline models are listed as 'GPT-5', 'GPT-4.1', 'Claude-Sonnet-4', 'Gemini-2.5-pro' without API versions or snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 (Figures 6-8) provides detailed system prompts for all agents: BlueprintAgent, UserAgent, AssistantAgent, ObservationAgent, and QualityAgent. These are actual prompt text, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.2 reports training hyperparameters: global batch size 16, learning rate 1e-5 with cosine decay, 50 warmup steps, sequence length 65,536 tokens, BF16 precision, Flash Attention v2. However, inference-time parameters (temperature, top-p) for the teacher model during synthesis are not stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent synthesis pipeline is described in detail in Section 3: BlueprintAgent, UserAgent, AssistantAgent, ObservationAgent, QualityAgent roles, Dynamic Schema Locking mechanism, Global Memory feedback loop, and adaptive evolution. Figure 2 provides an architectural overview."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Appendix C.1 document the data pipeline: 30,000 raw MCP tool definitions → encoding with Qwen-Embedding-0.6B → two-level K-Means clustering (Ndom=10, Ncls=5) → greedy selection of 6,944 Business Clusters → trajectory synthesis → quality filtering via QualityAgent."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, discussing compute costs of the multi-agent pipeline and the restriction to text-based tool calls and Python execution (not multi-modal)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: (1) per-trajectory API cost and latency are higher than lightweight baselines due to the multi-agent pipeline, and (2) the interaction scope is limited to text-based tool calls and Python execution, excluding graphical/multi-modal agents. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Limitations section mentions text-only scope and compute costs, it does not explicitly state what the results do NOT show. For example, it does not explicitly bound claims to the tested model family (Qwen), the specific teacher model, or the specific benchmarks used. The broad claims in the conclusion ('establishes that synthetic data pipelines must be measured and directed') are not bounded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the raw MCP tool definitions, the synthesized trajectories, nor the intermediate data are available for download. The paper promises future release but provides no links."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Appendix C describe the data collection: 30,000 raw MCP-compliant tool definitions were collected, organized into Business Clusters via embedding and clustering, then filtered via greedy selection to 6,944 clusters. Trajectories were synthesized using Qwen3-Max as teacher."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. All data is synthetically generated from MCP tool definitions and LLM-based trajectory synthesis."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from raw tool definitions to final training data is documented: tool collection (30,000) → embedding → K-Means clustering → greedy selection (6,944 clusters) → Blueprint-driven synthesis → quality filtering → final trajectories (500 or 5,000 samples). Appendix C provides additional detail."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section listing grants or sponsors is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: SIAT (CAS), Qoder, Alibaba Group, UNSW Sydney, and SUAT. The connection between authors and Qoder/Alibaba is visible."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess independence. Authors from Qoder and Alibaba Group could have financial interest in demonstrating the value of their training framework, but this is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Authors are affiliated with Qoder (a company) and Alibaba Group, which could have financial interests in the outcomes."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates fine-tuned models on benchmarks but does not state the training data cutoff for the base Qwen3-Coder model or the Qwen3-Max teacher model used for synthesis."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the base model (Qwen3-Coder) may have seen the benchmark data (BFCL, BIRD, etc.) during pre-training. The paper uses SWE-rebench which is designed for decontamination, but this is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several benchmarks used (BFCL, BIRD) were published before the likely training cutoff of Qwen3-Coder. LiveCodeBench/CodeCI is designed for contamination-free evaluation (noted in the citation), but the paper does not discuss contamination risk for any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. All experiments use synthetic data and automated benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The Ethical Considerations section explicitly states 'it did not involve human subjects, crowdsourcing, or personally identifiable information.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving human assignment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or evaluators requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported for either the synthesis pipeline or the evaluation runs. The Limitations section acknowledges 'per-trajectory API cost and latency are higher than lightweight baselines' but provides no numbers."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions training on '8 × 80GB GPUs' per node but does not state how many nodes, total GPU hours, total API spend for synthesis with Qwen3-Max, or wall-clock time for training or evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "TDScaling with 500 samples enables Qwen3-Coder-30B-A3B to reach 36.66% on BFCL Multi-turn, exceeding the much larger Qwen3-Coder-480B-A35B-Instruct (35.91%).",
       "evidence": "Table 1 shows TDScaling (500 Samples) at 36.66% vs. Qwen3-Coder-480B at 35.91% on BFCL Multi-turn.",
       "supported": "moderate"
     },
     {
       "claim": "TDScaling is the only method that achieves comprehensive improvements across all coding benchmarks without negative transfer.",
       "evidence": "Table 2 shows TDScaling with positive gains across all metrics (+1.92 RebenchT OH, +7.72 Qod, +4.00 CodeCI, +2.35 BIRD), while all baselines show negative transfer on at least some metrics.",
       "supported": "moderate"
     },
     {
       "claim": "Diversity scaling achieves a substantially higher performance ceiling than quantity scaling.",
       "evidence": "Figure 4 shows TDScaling performance increasing from ~36.66% at 500 samples to 40.44% at 5,000 samples, while baselines exhibit inverse scaling or stagnation.",
       "supported": "moderate"
     },
     {
       "claim": "The Code Tool acts as a regularizer against catastrophic forgetting of intrinsic coding capabilities.",
       "evidence": "Table 3 ablation: w/o Code Tool scores 37.56% on BFCL but drops to 41.58% on BIRD (vs. 43.83% full model), supporting the regularization claim.",
       "supported": "moderate"
     },
     {
       "claim": "TDScaling's synthesized data achieves higher Domain Entropy (4.25 vs. 2.15) and Reasoning Mode Entropy (8.97 vs. 5.42) compared to baseline synthesis.",
       "evidence": "Figure 3 reports these entropy scores comparing TDScaling vs. baseline datasets.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "TDScaling proposes shifting synthetic data scaling for code agents from quantity to diversity, using Business Cluster sampling, adaptive evolution guided by entropy/complexity metrics, and a sandboxed Code Tool as regularizer. With only 500 training samples, TDScaling enables a 30B-parameter model to match or exceed a 480B-parameter model on tool-use benchmarks (BFCL). Unlike competing tool-learning methods that suffer negative transfer on coding tasks, TDScaling improves both tool-use generalization and inherent coding proficiency across five benchmarks.",
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All results are reported as single point estimates without confidence intervals, error bars, significance tests, or variance across runs. Given that fine-tuning and benchmark evaluation can have high variance, the reported differences (e.g., 36.66% vs. 35.91% on BFCL) could be within noise."
     },
     {
       "flag": "Potential conflict of interest",
       "detail": "Authors from Qoder and Alibaba Group evaluate a training framework that could be commercially valuable. The base model (Qwen3-Coder) is from the Qwen family associated with Alibaba. No funding disclosure or competing interests statement is provided."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "Baselines (APIGen-MT, TOUCAN, Simia) are evaluated at 5,000 samples described as 'maximum common data availability,' but TDScaling uses a proprietary teacher (Qwen3-Max) and custom synthesis pipeline. The quality of the teacher model and synthesis process is a major confound not controlled for across methods."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Several benchmarks (BFCL, BIRD) predate the likely training cutoff of Qwen3-Coder. The paper does not discuss whether the base model may have already seen benchmark data during pre-training, which could inflate reported improvements."
     },
     {
       "flag": "Artifacts not released",
       "detail": "Despite claims of establishing 'a resource-efficient paradigm,' neither the code, synthesized data, nor trained models are released. The promise of future release is hedged with 'subject to licensing, privacy, and internal policy constraints.'"
     }
   ],
   "cited_papers": [
     {
       "title": "Gorilla: Large language model connected with massive APIs",
       "authors": ["Shishir G Patil", "Tianjun Zhang", "Xin Wang", "Joseph E Gonzalez"],
       "year": 2024,
       "relevance": "Defines the BFCL benchmark used for evaluation and represents prior work on connecting LLMs with APIs."
     },
     {
       "title": "ToolLLM: Facilitating large language models to master 16000+ real-world APIs",
       "authors": ["Yujia Qin", "Shihao Liang", "Yining Ye"],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Key prior work on scaling tool-use training data; represents the quantity-scaling paradigm that TDScaling critiques."
     },
     {
       "title": "APIGen-MT: Agentic pipeline for multi-turn data generation via simulated agent-human interplay",
       "authors": ["Akshara Prabhakar", "Zuxin Liu", "Ming Zhu"],
       "year": 2025,
       "arxiv_id": "2504.03601",
       "relevance": "A directly compared baseline for tool-use trajectory synthesis with Blueprint-driven mechanisms."
     },
     {
       "title": "TOUCAN: Synthesizing 1.5M tool-agentic data from real-world MCP environments",
       "authors": ["Zhangchen Xu", "Adriana Meza Soria", "Shawn Tan"],
       "year": 2025,
       "arxiv_id": "2510.01179",
       "relevance": "A directly compared baseline that generates tool-use data from real MCP environments at scale."
     },
     {
       "title": "Simia: Simulating environments with reasoning models for agent training",
       "authors": ["Yuetai Li", "Huseyin A Inan", "Xiang Yue"],
       "year": 2025,
       "arxiv_id": "2511.01824",
       "relevance": "A directly compared baseline that uses reasoning models to simulate environments for agent training."
     },
     {
       "title": "SWE-rebench: An automated pipeline for task collection and decontaminated evaluation of software engineering agents",
       "authors": ["Ibragim Badertdinov", "Alexander Golubev"],
       "year": 2025,
       "arxiv_id": "2505.20411",
       "relevance": "Benchmark used for evaluation; addresses decontamination in software engineering agent evaluation."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Benchmark used for evaluation; designed for contamination-free code evaluation."
     },
     {
       "title": "Can LLM already serve as a database interface? A big bench for large-scale database grounded text-to-SQLs",
       "authors": ["Jinyang Li", "Binyuan Hui", "Ge Qu"],
       "year": 2023,
       "relevance": "The BIRD benchmark used for text-to-SQL evaluation in this paper."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2022,
       "relevance": "Foundational work on combining reasoning and action in language models, which this paper builds upon for agentic code agents."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2023,
       "relevance": "Seminal work on tool-augmented language models, foundational to the tool-learning paradigm this paper extends."
     },
     {
       "title": "DeepSeek-V3.2: Pushing the frontier of open large language models",
       "authors": ["Aixin Liu"],
       "year": 2025,
       "arxiv_id": "2512.02556",
       "relevance": "Contemporary open-source model that strengthened tool-use capability; compared as a baseline achieving top performance."
     },
     {
       "title": "A survey on code generation with LLM-based agents",
       "authors": ["Yihong Dong", "Xue Jiang", "Jiaru Qian"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Recent survey of code generation agents that contextualizes tool integration in code agents."
     }
   ]
 }