ai-research-survey

scan.json (20029B)

---

 {
   "paper": {
     "title": "Curriculum Guided Massive Multi Agent System Solving for Robust Long Horizon Tasks",
     "authors": ["Indrajit Kar", "Kalathur Chenchu Kishore Kumar"],
     "year": 2025,
     "venue": "Wipro Innovation Networks (preprint)"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset or benchmark data is released. The Tower of Hanoi benchmark is custom-built but not shared."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. The paper mentions Mistral and DeepSeek but gives no version details or software dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1.1, 1.2, 1.7, and 1.8 show 95% confidence intervals on run-rate and cumulative regret curves."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Thompson Sampling outperforms UCB and ε-Greedy but provides no statistical significance tests to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes (Cohen's d, percentage improvement with baseline context, etc.) are reported. Results are shown only in figures without quantified differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of experimental runs, the choice of 2000 ticks, or the 64×64 grid size. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The 95% confidence interval bands in the run-rate and regret figures implicitly convey variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares Thompson Sampling against UCB and ε-Greedy (Figures 1.1, 1.2), and compares NLL+Curriculum vs. Curriculum-only vs. Base RL (Figures 1.7, 1.8). Figure 1.9 compares against traditional ML, deep learning, and RL-only agents."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are all bandit algorithm variants (UCB, ε-Greedy) and generic ML/DL/RL agents. No comparison against contemporary multi-agent LLM frameworks like MAKER [19], which is discussed in the related work, or other recent systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares NLL+Curriculum learning vs. Curriculum-only (Figure 1.7) vs. Base RL without NLL or curriculum (Figure 1.8), effectively ablating the NLL and curriculum components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports run-rate (stage progression speed), cumulative regret, NLL, competence scores, and oracle usage as evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for this simulated multi-agent system benchmark evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "The system is evaluated on a simulated Tower of Hanoi task, not on a train/test split dataset. The concept of held-out test sets does not structurally apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by curriculum stage (Stages 1-4), by spatial region (arm selections), and by algorithm type. Table 1 shows per-stage details."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1.9 shows failure cases for traditional ML, deep learning, and base RL agents. The paper also discusses posterior lock-in as a limitation of Thompson Sampling (Section 5)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that traditional ML and DL pixel agents fail entirely (Figure 1.9a, 1.9b), and that base RL without curriculum shows limited success (Figure 1.9c). Posterior lock-in of suboptimal arms under Thompson Sampling is also noted."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims improved stability, reduced oracle usage, and stronger long-range reasoning. The figures show confidence intervals for stability, oracle escalation mechanisms are demonstrated, and curriculum progression results support the long-range reasoning claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims (e.g., 'NLL-based calibration prevents premature advancement', 'Thompson Sampling achieves the lowest cost because it reaches competence earliest') but the experimental design does not control for all variables simultaneously. The ablation covers some components but the simulation itself conflates multiple design choices."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and discussion claim applicability to 'robotic manipulation and planning tasks' and 'automotive factory AI workflows' (Section 6), but the evaluation is solely on a simulated Tower of Hanoi benchmark. The title says 'Robust Long Horizon Tasks' but only one task is tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider whether the improvements stem from the curriculum structure, the specific bandit algorithm, the NLL metric, or the simulation parameters."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Mistral' for the SLM and 'DeepSeek' for the Oracle but provides no version numbers, model sizes, or snapshot dates for either."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts in natural language (e.g., 'pixel:10,50,cat:2') but does not provide the full prompt text sent to either Mistral or DeepSeek."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters like η (competence learning rate), α (pity bonus coefficient), γ (SLM confidence weight), θ (verifier threshold), τ_green (green-pixel threshold), and the bandit weight coefficients w_c and w_n are defined symbolically but their actual numerical values used in experiments are not stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail across Sections 4.2-4.10: the PixelGrid substrate, Verifier module, Oracle escalation via Router, Curriculum Manager, and Thompson Sampling meta-controller are all specified with mathematical formulations."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the Tower of Hanoi moves are generated, how the spiral mapping was validated, or what preprocessing occurs between raw simulation output and the reported figures."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The brief mention of posterior lock-in (Section 5) is a property of Thompson Sampling, not a limitation of the study design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It generalizes freely from Tower of Hanoi to factory automation without bounding the scope of its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw simulation data, logs, or intermediate results are made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the experimental setup: 64×64 grid, 4096 agents, 20 ticks/second, Tower of Hanoi benchmark with spiral mapping. The simulation procedure is described in detail."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; the study uses a simulated multi-agent system."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how simulation outputs are aggregated into the reported figures, how many runs were performed, or how confidence intervals were computed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The authors are affiliated with Wipro Innovation Networks but no funding acknowledgment is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors list their affiliation as Wipro Innovation Networks on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Wipro as a technology company may have commercial interest in multi-agent AI systems."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Mistral and DeepSeek models but does not state their training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Tower of Hanoi task structure or solutions could be in the training data of Mistral or DeepSeek."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Tower of Hanoi is a well-known classical problem. The paper does not discuss whether LLMs may have seen Tower of Hanoi solutions during training, which would undermine claims about reasoning capability."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses reducing oracle usage conceptually but does not report actual inference costs, API costs, tokens consumed, or wall-clock time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, API spend, or hardware specifications are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Thompson Sampling achieves the fastest run-rate and lowest cumulative regret compared to UCB and ε-Greedy for curriculum management.",
       "evidence": "Figures 1.1 and 1.2 show run-rate trajectories and cumulative regret with 95% confidence intervals across the three algorithms (Section 5).",
       "supported": "moderate"
     },
     {
       "claim": "NLL-augmented curriculum learning outperforms curriculum-only and base RL approaches.",
       "evidence": "Figures 1.1, 1.7, and 1.8 compare NLL+Curriculum, Curriculum-only, and Base RL run-rate trajectories. Figure 1.9 shows qualitative success/failure comparisons.",
       "supported": "moderate"
     },
     {
       "claim": "The framework can be transferred to robotic manufacturing systems and automotive factory AI workflows.",
       "evidence": "Section 6 (Discussion) draws an analogy between Tower of Hanoi dependency structure and factory workflows. No empirical evidence for this transfer is provided.",
       "supported": "unsupported"
     },
     {
       "claim": "Selective oracle escalation reduces unnecessary oracle usage while maintaining correctness.",
       "evidence": "The Verifier module (Section 4.6) and Oracle escalation (Section 4.7) describe the mechanism, but no quantitative measurement of oracle usage reduction is reported.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes a hierarchical multi-agent architecture using a 64x64 grid of lightweight agents (Mistral SLM) with selective escalation to a DeepSeek oracle, guided by a spatial curriculum and Thompson Sampling curriculum manager. NLL-based calibration is integrated into the reward signal to ensure agents advance only when both accurate and confident. On a spatially-grounded Tower of Hanoi benchmark, Thompson Sampling outperforms UCB and ε-Greedy in run-rate and cumulative regret, and the full NLL+curriculum system outperforms curriculum-only and base RL variants.",
   "red_flags": [
     {
       "flag": "No reproducibility artifacts",
       "detail": "No code, data, environment specs, or reproduction instructions are provided. Key hyperparameter values are defined symbolically but never given numerical values."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "The paper claims applicability to robotic manufacturing and automotive factory workflows (Section 6) based solely on a simulated Tower of Hanoi benchmark. The title claims 'Robust Long Horizon Tasks' but only one task is evaluated."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "Tower of Hanoi is a classical problem whose solutions are widely available online. The paper does not discuss whether Mistral or DeepSeek have seen Tower of Hanoi solutions during training, which could confound claims about reasoning capability."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated discussion of limitations, threats to validity, or scope boundaries."
     },
     {
       "flag": "Unspecified model versions and hyperparameters",
       "detail": "Mistral and DeepSeek are referenced without version numbers. All hyperparameters (η, α, γ, θ, τ, weights) are defined symbolically but actual values used in experiments are not reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Solving a Million-Step LLM Task with Zero Errors (MAKER)",
       "authors": ["E. Meyerson", "G. Paolo", "R. Dailey", "H. Shahrzad", "O. Francon", "C. F. Hayes", "X. Qiu", "B. Hodjat", "R. Miikkulainen"],
       "year": 2025,
       "arxiv_id": "2511.09030",
       "relevance": "Directly relevant as a competing long-horizon LLM task-solving framework."
     },
     {
       "title": "Measuring AI Ability to Complete Long Tasks",
       "authors": ["T. Kwa", "B. West", "J. Becker"],
       "year": 2025,
       "relevance": "Benchmark for evaluating AI agents on long-horizon tasks, directly relevant to the survey scope."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["R. Schaeffer", "B. Miranda", "S. Koyejo"],
       "year": 2023,
       "relevance": "Foundational work questioning LLM capability claims, relevant to methodology quality assessment."
     },
     {
       "title": "Faith and Fate: Limits of Transformers on Compositionality",
       "authors": ["N. Dziri", "X. Lu", "M. Sclar"],
       "year": 2023,
       "relevance": "Analyzes fundamental limitations of transformers on compositional reasoning tasks like Tower of Hanoi."
     },
     {
       "title": "The Illusion of Thinking: Understanding the Strengths and Limitations of Reasoning Models via the Lens of Problem Complexity",
       "authors": ["P. Shojaee", "I. Mirzadeh", "K. Alizadeh"],
       "year": 2025,
       "arxiv_id": "2506.06941",
       "relevance": "Evaluates reasoning model capabilities and limitations, relevant to LLM capability assessment."
     },
     {
       "title": "OPTIMA: Optimizing Effectiveness and Efficiency for LLM-Based Multi-Agent System",
       "authors": ["W. Chen", "J. Yuan", "C. Qian"],
       "year": 2025,
       "relevance": "Multi-agent LLM system optimization, directly in survey scope."
     },
     {
       "title": "Plan-and-Act: Improving Planning of Agents for Long-Horizon Tasks",
       "authors": ["L. E. Erdogan", "H. Furuta", "S. Kim"],
       "year": 2025,
       "relevance": "Long-horizon agent planning at ICML, directly relevant to agentic AI capabilities."
     },
     {
       "title": "MAS Failure Taxonomy: Failure Modes in LLM-Based Multi-Agent Systems",
       "authors": ["X. Zhang"],
       "year": 2025,
       "relevance": "Taxonomy of multi-agent system failures, relevant to AI safety and reliability."
     },
     {
       "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
       "authors": ["X. Wang", "J. Wei", "D. Schuurmans", "Q. Le", "E. Chi"],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Foundational LLM reasoning technique relevant to the survey's coverage of reasoning methods."
     }
   ]
 }