ai-research-survey

scan.json (25540B)

---

 {
   "paper": {
     "title": "Jenius Agent: Towards Experience-Driven Accuracy Optimization in Real-World Scenarios",
     "authors": ["Defei Xia", "Bingfeng Pi", "Shenbin Zhang", "Song Hua", "Yunfei Wei", "Lei Zuo"],
     "year": 2026,
     "venue": "arXiv (submitted to MLSys 2025)",
     "arxiv_id": "2601.01857",
     "doi": "10.48550/arXiv.2601.01857"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Jenius-Agent integrates adaptive prompt generation, context-aware tool orchestration, and hierarchical memory management into a ReAct-style agent. On their custom Jenius-bench (850 multi-turn tasks), the full system achieves 76.5% task completion vs 56.6% baseline (35% relative gain). Token consumption is reduced by over 60%. On APIGen, gains are marginal (81.5% to 85.0%), suggesting the benchmark saturates. The system is deployed at jenius.cn with 265 DAU after 6 months.",
   "claims": [
     {
       "claim": "Up to 35% relative gain in task completion rate over the base agent on Jenius-bench",
       "evidence": "Table 3: Base TCR=0.5659, Jenius TCR=0.7647 (Section 4.4.1). This is a 35% relative improvement.",
       "supported": "strong"
     },
     {
       "claim": "Over 60% reduction in token consumption compared to baseline",
       "evidence": "Figure 6 and Section 4.4.3: Jenius-bench usage decreases from 9.27M to 3.65M tokens.",
       "supported": "strong"
     },
     {
       "claim": "Adaptive prompt generation is the single largest contributor, improving TCR from 0.5659 to 0.7271 (16%)",
       "evidence": "Table 3: Base→B-P comparison. Subsequent modules (tool orchestration, memory) add smaller incremental gains.",
       "supported": "moderate"
     },
     {
       "claim": "APIGen benchmark saturates, showing only marginal improvements from optimization modules",
       "evidence": "Table 2: Base TCR=0.8150, Jenius TCR=0.8500. Section 4.4.1 attributes this to pre-training coverage.",
       "supported": "moderate"
     },
     {
       "claim": "8-10% improvement in correctness and fluency over baseline on CRCFF metrics",
       "evidence": "Table 4: Correctness improves from 0.6741 to 0.7580 (Qwen) and 0.7890 to 0.8350 (DeepSeek).",
       "supported": "weak"
     },
     {
       "claim": "System deployed at scale with 265 DAU after 6 months",
       "evidence": "Section 5.2: DAU grew from 42 to 265, MAU of 1,277 in first month, spanning 34 countries.",
       "supported": "moderate"
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL is provided. The paper mentions jenius.cn as a deployed product but releases no code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Jenius-bench is described in detail but no download link or repository is provided. APIGen is public, but the custom 800-sample subset and noise-injection setup are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, requirements file, or dependency list is provided. The paper mentions Alibaba Cloud and Kubernetes but no software versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The system is described architecturally but there are no steps to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2-4 report only point estimates. No confidence intervals, error bars, or ± notation anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements across multiple configurations but performs no statistical significance tests. All comparisons are raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context: e.g., TCR from 0.5659 to 0.7271 (↑16%), correctness from 0.6741 to 0.7580 (↑5.2%), and token reduction from 9.27M to 3.65M (>60%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "APIGen uses 800 sampled instances and Jenius-bench has 850 samples. No justification is given for these sizes or any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares four progressively enhanced configurations: Base (ReAct), B-P (+ prompt), B-PT (+ tool), and Jenius (full system). Section 4.3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "All baselines are internal ablations of the same system. No comparison against any external agent framework (e.g., AutoGen, LangChain, or other published agent systems)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The four configurations (Base, B-P, B-PT, Jenius) form a progressive ablation, isolating the contribution of each module. Tables 2-3."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 4T metrics (TCR, TFR, TIR, TPS) for execution fidelity, CRCFF (5 dimensions) for output quality, and token consumption. Section 4.1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Output quality is evaluated entirely by LLM judges (Qwen and DeepSeek). No human evaluation of system outputs is performed. The Jenius-bench annotations are human-curated ground truth, but evaluation of the system's outputs is automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No mention of train/dev/test splits. The paper evaluates on the full APIGen sample and Jenius-bench without discussing whether any data was used for development or tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Jenius-bench covers 38 tool categories and APIGen has 21 categories, but no per-category performance breakdown is provided. Only aggregate results are reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses several real-world failure modes: spurious tool calls, URL reader failures, inappropriate triggering of expensive tools, and retry loops. Section 4.4.1 discusses TFR and TIR as failure metrics."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that full Jenius increased TFR on Jenius-bench from 0.0329 to 0.0753 (Section 4.4.1, Table 3), and that memory management showed marginal gains on DeepSeek evaluator (Table 4). Section 6 acknowledges limitations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'up to 35% relative gain' and 'reduced token consumption' are supported by Tables 2-3 and Figure 6. The abstract appropriately hedges with 'up to'."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (each module 'improves' performance) are supported by controlled ablation: each configuration adds exactly one module to the previous, enabling attribution. Section 4.3."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Real-World Scenarios' broadly, but evaluation is limited to their own Jenius-bench and APIGen. The paper does not bound claims to these specific benchmarks, and the abstract frames this as a general 'scalable solution for robust, protocol-compatible autonomous agents.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed improvements are discussed. For example, the prompt optimization module could simply be providing more information rather than 'adaptive' optimization. The improvements could stem from better prompt engineering rather than the proposed architecture."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses TCR and CRCFF (LLM-judged scores) as proxies for agent quality and 'accuracy optimization' but does not discuss the gap between these metrics and actual user satisfaction or real-world task success. Section 5 mentions user engagement but does not connect deployment metrics to evaluation metrics."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Qwen3-Embedding for tool retrieval and Qwen/DeepSeek as LLM evaluators, but never specifies which LLM powers the agent itself, nor provides model versions/snapshot dates for any model used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt design principles (Section 3.1) at a high level but does not provide actual prompt text used in experiments. Only design patterns and categories are described."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported: no temperature, top-p, max tokens, embedding dimensions, threshold K for memory summarization, or M for top-M retrieval."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-module scaffolding is described in detail: adaptive prompt generation (Section 3.1), tool orchestration with embedding similarity and inflection-point filtering (Section 3.2), and hierarchical memory management with dialogue-level alignment and session-level summarization (Section 3.3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "For Jenius-bench: the paper says 850 'human-curated samples' with 'manual review and annotation by domain experts' but does not describe the curation pipeline, filtering criteria, or annotation guidelines. For APIGen: 800 instances were 'sampled' with no description of sampling method."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section 6 (Discussion and Conclusion) briefly mentions 'incomplete capture of hidden reasoning steps' and 'multiple valid tool-use paths' but this is embedded in the conclusion, not a substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The brief mention in Section 6 is generic ('highlighting limitations') without identifying specific threats like evaluator bias, benchmark representativeness, or internal validity concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show, what task types are excluded, or what settings the framework has not been tested in."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither Jenius-bench data nor experimental results (individual task outcomes) are available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Jenius-bench is described as 'derived from real user-agent interactions' with 'manual review,' but the collection procedure, time period, and selection criteria are not specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation. Jenius-bench is derived from logged interactions, not a participant study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw user interactions to the final 850-sample benchmark is not documented. No filtering stages, exclusion counts, or annotation procedures are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with 'Tianju Dihe (Suzhou) Technology Co., Ltd.' which operates the Jenius product being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors work at the company that develops and commercializes Jenius. The funder (the company itself) has a direct financial interest in positive results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. The authors work for the company whose product they evaluate, but this conflict is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The underlying LLM powering the agent is not even named, let alone its training cutoff. APIGen was published in 2024 and could be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the agent's LLM may have seen APIGen examples or similar tasks during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "APIGen is a public benchmark (NeurIPS 2024). No contamination analysis is performed. Jenius-bench is proprietary so contamination risk is lower, but this is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token consumption is reported in Figure 6: from 9.27M (Base) to 3.65M (Jenius) on Jenius-bench. Section 4.4.3 discusses token efficiency in detail."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, API cost in dollars, or hardware specifications for experiments are stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating whether they are from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The inflection-point threshold N=10 was chosen empirically (Section 3.2) but the search process is not documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The progressive ablation design (Base→B-P→B-PT→Jenius) shows all configurations, but no justification for how specific sub-choices (e.g., N=10, threshold K) were selected."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (Jenius) on their own benchmark (Jenius-bench) without acknowledging this bias. Section 4.4 notes the within-framework comparison design but frames it positively rather than as a limitation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 and Section 4.4.3 report token consumption alongside performance for each configuration, enabling performance-per-token analysis."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Jenius-bench or APIGen actually measure what they claim. The paper does not question construct validity of TCR as a proxy for agent quality or CRCFF as output quality."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates Jenius as a bundled system; the scaffold IS the thing being tested. No cross-model comparison is made that would require isolating scaffold effects."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. APIGen was published in 2024 and could be in the training data of models used. No cutoff dates are stated."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information, e.g., whether tool descriptions in Jenius-bench contain hints toward the correct answer."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between Jenius-bench tasks, or whether some tasks share structural similarities that could inflate performance."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied to either benchmark."
       }
     }
   },
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors work at Tianju Dihe, which develops and commercializes Jenius. They evaluate their own system on their own proprietary benchmark (Jenius-bench) with no external validation, no independent evaluators, and no competing interests disclosure."
     },
     {
       "flag": "Proprietary benchmark with no release",
       "detail": "Jenius-bench (850 tasks) is the primary evaluation dataset but is not released. The data collection pipeline, annotation guidelines, and curation criteria are undocumented. Results cannot be independently verified."
     },
     {
       "flag": "No external baselines",
       "detail": "All comparisons are internal ablations (Base, B-P, B-PT, Jenius). No comparison against any published agent framework (AutoGen, LangChain, etc.) despite these being discussed in Related Work."
     },
     {
       "flag": "LLM-as-judge without validation",
       "detail": "CRCFF quality metrics are evaluated entirely by LLM judges (Qwen and DeepSeek). No human validation of the LLM judges' reliability or correlation with human judgment is provided."
     },
     {
       "flag": "No statistical rigor",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, significance tests, or variance across seeds. Marginal differences (e.g., 0.7494 vs 0.7647) may not be meaningful."
     },
     {
       "flag": "Undisclosed base model",
       "detail": "The LLM powering the agent is never identified. Without knowing the model, it is impossible to assess contamination risk, reproduce results, or compare fairly with other systems."
     },
     {
       "flag": "Tiny deployment scale presented as validation",
       "detail": "Section 5 presents 265 DAU and 1,277 MAU as evidence of real-world effectiveness, but these are very small numbers. No user satisfaction data, A/B test results, or outcome metrics from deployment are provided."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agent reasoning paradigm (ReAct) that Jenius builds upon."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "authors": ["Qingyun Wu"],
       "relevance": "Major multi-agent framework discussed as a related approach with different execution assumptions."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Directly relevant to understanding agent failure modes, which Jenius claims to address."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["Abhimanyu Dubey"],
       "year": 2024,
       "relevance": "Major open LLM family relevant to LLM-based agent capability."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick"],
       "year": 2023,
       "relevance": "Foundational work on LLM tool use that Jenius's tool orchestration extends."
     },
     {
       "title": "APIGen: Automated Pipeline for Generating Verifiable and Diverse Function-Calling Datasets",
       "authors": ["Zuxin Liu"],
       "year": 2024,
       "relevance": "Public benchmark used in this paper's evaluation."
     },
     {
       "title": "MCP-Zero: Proactive Toolchain Construction for LLM Agents from Scratch",
       "authors": ["Xiang Fei"],
       "year": 2025,
       "arxiv_id": "2506.01056",
       "relevance": "Adaptive tool discovery approach related to Jenius's tool orchestration module."
     },
     {
       "title": "DSPy: Compiling Declarative Language Model Calls into State-of-the-Art Pipelines",
       "authors": ["Omar Khattab"],
       "relevance": "Modular prompt compilation framework relevant to prompt optimization research."
     },
     {
       "title": "The Rise and Potential of Large Language Model Based Agents: A Survey",
       "authors": ["Zhiheng Xi"],
       "year": 2025,
       "relevance": "Comprehensive survey of LLM-based agents relevant to the survey scope."
     },
     {
       "title": "The Landscape of Emerging AI Agent Architectures for Reasoning, Planning, and Tool Calling: A Survey",
       "authors": ["Tula Masterman"],
       "year": 2024,
       "arxiv_id": "2404.11584",
       "relevance": "Survey of agent architectures for reasoning and tool calling."
     },
     {
       "title": "From RAG to Multi-Agent Systems: A Survey of Modern Approaches in LLM Development",
       "authors": ["Gabriel de Araujo e Aquino"],
       "year": 2025,
       "relevance": "Survey covering RAG and multi-agent system evolution relevant to agent development."
     }
   ]
 }