ai-research-survey

scan.json (25954B)

---

 {
   "paper": {
     "title": "Sherlock: Reliable and Efficient Agentic Workflow Execution",
     "authors": ["Yeonju Ro", "Haoran Qiu", "Íñigo Goiri", "Rodrigo Fonseca", "Ricardo Bianchini", "Aditya Akella", "Zhangyang Wang", "Mattan Erez", "Esha Choukse"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.00330",
     "doi": "10.48550/arXiv.2511.00330"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL, GitHub link, or archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: CoTCollection, OMEGA, LiveCodeBench, HumanEval, MBPP, HotpotQA, DROP, GSM8k, MATH-500, GTA. All are referenced with citations or HuggingFace links."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (8×A100 80GB, vLLM serving) and model IDs are given (Table 2), but no requirements.txt, Dockerfile, or library version list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The paper describes the methodology but does not provide commands or procedures to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Figures 12-14 and Table 3 report point estimates only. No confidence intervals, error bars, or ± notation on the key accuracy, latency, or cost numbers."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Sherlock 'consistently achieves higher accuracy' and 'delivers an 18.3% accuracy gain' but no statistical significance tests are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are given with baseline context: '18.3% accuracy gain on average', 'up to 48.7% execution time reduction', '26.0% cost reduction'. Table 3 provides detailed latency reduction percentages per benchmark and percentile."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the number of benchmark examples used. The paper mentions '100+ different graphs and 15K+ execution traces' for fault injection but does not justify these quantities."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: non-verifying baseline, random and even verifier placement, static single-verifier configurations, AFlow (Monte Carlo search), tabular approach, and an oracle (§8)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "AFlow (Zhang et al., 2024a) and Flow planner (Niu et al., 2025) are contemporary. The verifier methods (Self-Refine, Debate, LLM-as-a-Judge) are all from 2023-2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Each component is evaluated in isolation: verifier placement (§8.2, Figure 12), verifier selection (§8.3, Figures 13-14), and speculative execution (§8.4, Figure 15, Table 3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metric dimensions are used: accuracy (percentage points, pass@1), latency (Texec and Tvrf at multiple percentiles), and cost (normalized cost per 1K problems)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using benchmark metrics. No human evaluation of workflow output quality is included, though it could be relevant for instruction-following tasks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets are used (CoTCollection, OMEGA, LiveCodeBench). The verifier selector is trained on microbenchmark data and evaluated on the agentic benchmarks (§8.3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task category (instruction, code, math, tool) in Figures 3-5, and by benchmark (CoTCollection, LiveCodeBench, OMEGA) in Figures 12, 14, and Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where lightweight similarity metrics fail (code and math tasks, Table 5, AUC ≈ 0.5), cases where verification reduces accuracy (Figure 3), and rollback scenarios (§7.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: verification can reduce accuracy in some cases (Figure 3, e.g., Self-Refine on Code), similarity metrics fail for code/math tasks (Table 5), and fan-out shows no correlation with vulnerability (§5.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '18.3% accuracy gain' (supported by Figure 14 comparisons), 'up to 48.7% execution time reduction' (Table 3, LiveCodeBench Tvrf mean), and '26.0% cost reduction compared to Monte Carlo search' (Figure 14 vs AFlow) are all supported by the experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses controlled ablation experiments isolating each component (placement in §8.2, selection in §8.3, speculation in §8.4). Each comparison changes one variable while holding others constant, which is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Reliable and Efficient Agentic Workflow Execution' broadly, but results are on three specific benchmarks with one base model (Llama-3.1-8B). The paper notes domain on-boarding is needed but does not explicitly bound generalizations to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, could the gains come from the specific model pairing rather than the framework? Could the topological policy be an artifact of the workflow generator used?"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy (benchmark-specific metrics), latency (wall-clock time), and cost (dollar amounts via cost model). Claims match measurement granularity — e.g., 'accuracy gain on CoTCollection' rather than broader 'reliability' claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact HuggingFace model IDs are provided: 'meta-llama/Llama-3.1-8B-Instruct', 'meta-llama/Llama-3.3-70B-Instruct', 'Qwen/Qwen2.5-7B-Instruct', 'AtlaAI/Selene-1-Mini-Llama-3.1-8B' (Table 2, §8.1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendix D for the scorer (D.1), judge (D.2), majority vote (D.3), rollback (D.4), self-refine (D.5), debate rounds (D.6-D.7), and LLM planner (D.8)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for fault injection (§5.2), and λ is mentioned as a tunable parameter. However, GRPO training hyperparameters (learning rate, epochs, batch size), vLLM serving parameters, and sampling settings for the main executor are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic workflow structure is described in detail: workflow generation via Flow planner (§2.1), verifier placement policy (§5.4, Algorithm 1), verifier selector architecture (§6), and speculative execution runtime with state machine (§7, Appendix B)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The workflow generation process is documented (§8.1, Flow planner with customized prompts in Appendix D.8), fault injection methodology is detailed (§5.1-5.2, Table 1), and benchmark descriptions are provided (§8.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The paper goes directly from Evaluation (§8) to Related Work (§9) to Conclusion (§10) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what settings, model types, or workflow structures the results do NOT apply to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw execution traces, fault injection results, or training data for the verifier selector are released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Fault injection methodology is described in detail (§5.1-5.2): failure modes from Cemri et al. (2025), fault injection procedure, '100+ different graphs and 15K+ execution traces' generated from benchmarks."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from benchmark tasks → workflow generation (Flow planner) → fault injection → vulnerability estimation → verifier selector training is documented across §5-6. Table 1 shows failure mode frequencies sourced from prior empirical work."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Microsoft Azure Research, Microsoft Azure, and The University of Texas at Austin. The paper evaluates open-source models, not Microsoft products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. Given Microsoft Azure Research affiliation, corporate funding is implied but not stated. Microsoft has commercial interest in agentic workflow serving (Azure AI services), making funder independence unclear."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff for Llama-3.1-8B and other models used is not stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (HumanEval published 2021, CoTCollection 2023) may overlap with training data of the models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and other benchmarks predate the models' training. LiveCodeBench is designed to be contamination-free, but the paper does not discuss contamination risk for any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference cost is reported as normalized cost per 1K problems (Figure 13), average cost comparisons (Figure 14), and the cost model is detailed in Appendix A.2 including GPU cost formula and $13.60/hour rate."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified: '8×NVIDIA A100 GPUs (80 GB each)' with GPU allocation per model (Table 2). Cost is $13.60/hour. 15K+ execution traces generated for fault injection analysis."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. Temperature is fixed to 0 for fault injection (§5.2), but main results do not show seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs for main evaluation results is not stated. '15K+ execution traces' is mentioned for fault injection analysis but not for the end-to-end evaluation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for the verifier selector training (GRPO) or any other component."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows Pareto frontiers (Figure 13) and mentions λ as a tunable knob, but does not justify how specific configurations were selected for the end-to-end evaluation results."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (random placement, even placement, tabular approach) and do not acknowledge potential bias from evaluating their own system against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 13 shows the accuracy-cost tradeoff explicitly. The paper systematically compares methods at various cost levels and shows Sherlock follows the Pareto frontier."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CoTCollection, OMEGA, and LiveCodeBench adequately measure 'reliable and efficient agentic workflow execution' as claimed. The paper does not question construct validity of any benchmark."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All methods use the same Flow planner for workflow generation and the same base executor (Llama-3.1-8B-Instruct). Verifier placement is the only variable in §8.2, verifier selection in §8.3, and speculation in §8.4."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Llama-3.1's training data includes solutions to benchmark problems created before its training cutoff."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context or prompts."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test independence for the verifier selector (trained on microbenchmarks, tested on agentic benchmarks) or any other component."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied to any benchmark."
       }
     }
   },
   "claims": [
     {
       "claim": "Sherlock delivers an 18.3% accuracy gain on average across benchmarks compared to the non-verifying baseline.",
       "evidence": "End-to-end results in Figure 14 comparing baseline, AFlow, tabular, and Sherlock accuracy across CoTCollection, LiveCodeBench, and OMEGA.",
       "supported": "moderate"
     },
     {
       "claim": "Sherlock reduces workflow execution time by up to 48.7% over non-speculative execution.",
       "evidence": "Table 3 shows LiveCodeBench mean Tvrf reduction of 48.7%. CoTCollection Tvrf mean reduction is 21.9%, OMEGA is 30.6%.",
       "supported": "strong"
     },
     {
       "claim": "Sherlock lowers verification cost by 26.0% compared to Monte Carlo search-based method (AFlow).",
       "evidence": "Figure 14 (right panel) shows normalized cost comparison between AFlow and Sherlock across three benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "Terminal nodes are most vulnerable to faults, followed by initial nodes, while intermediate nodes contribute least to end-to-end failure propagation.",
       "evidence": "Figure 8 (left) shows vulnerability estimates by node position across CoTCollection, HumanEval, and OMEGA benchmarks, with terminal > root > middle.",
       "supported": "strong"
     },
     {
       "claim": "Fan-in degree shows strong positive correlation with node vulnerability.",
       "evidence": "Figure 8 (right) shows monotonically increasing vulnerability estimates (0.15 to 0.27 for CoTCollection, 0.14 to 0.21 for HumanEval) as fan-in increases from 1 to 5.",
       "supported": "strong"
     },
     {
       "claim": "Sherlock's verifier selector follows the Pareto frontier and approaches oracle performance.",
       "evidence": "Figure 13 shows Sherlock points on the accuracy-cost Pareto frontier, close to the oracle.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Sherlock presents a principled framework for selective verification in agentic workflows, using counterfactual fault injection to identify error-prone nodes and GRPO-trained verifier selection. Terminal and high-fan-in nodes are most vulnerable. Speculative execution reduces latency by up to 48.7% by overlapping verification with downstream computation, while the learned verifier selector achieves near-Pareto-optimal accuracy-cost tradeoffs across instruction-following, coding, math, and tool-use tasks.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or scope boundary discussion despite making broad claims about 'reliable and efficient agentic workflow execution.' This is a significant methodological omission."
     },
     {
       "flag": "No variance or error bars on main results",
       "detail": "All key accuracy, latency, and cost results are reported as point estimates without any uncertainty quantification. LLM outputs are stochastic, so single-run results may not be stable."
     },
     {
       "flag": "Company evaluating related capability",
       "detail": "Multiple authors are from Microsoft Azure Research, which has commercial interest in agentic workflow serving infrastructure. No funding or competing interests are disclosed."
     },
     {
       "flag": "Single base model",
       "detail": "All experiments use Llama-3.1-8B-Instruct as the base executor. Generalization of the topology-based vulnerability policy and verifier selector to other models or model sizes is not tested."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Key LLM verification method (self-reflection) that Sherlock builds upon and compares against."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba"],
       "year": 2023,
       "relevance": "Multi-agent debate verification method used as a baseline in Sherlock's verifier selection."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "arxiv_id": "2203.11171",
       "relevance": "Self-consistency verification method evaluated and compared in Sherlock's framework."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "LLM-as-a-Judge paradigm used as a verifier type in Sherlock."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Mike Z. Pan", "Shijie Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Provides the empirical failure mode distribution used to calibrate Sherlock's fault injection model."
     },
     {
       "title": "AFlow: Automating agentic workflow generation",
       "authors": ["Jiale Zhang", "Jiahui Xiang", "Zhiyuan Yu"],
       "year": 2024,
       "relevance": "Monte Carlo search-based workflow optimization baseline compared against Sherlock."
     },
     {
       "title": "Flow: Modularized agentic workflow automation",
       "authors": ["Boyu Niu", "Yifan Song", "Kaituo Lian"],
       "year": 2025,
       "relevance": "LLM planner used for workflow generation in Sherlock's evaluation."
     },
     {
       "title": "Speculative Actions: A lossless framework for faster agentic systems",
       "authors": ["Nan Ye", "Abhinav Ahuja", "Georgios Liargkovas"],
       "year": 2025,
       "arxiv_id": "2510.04371",
       "relevance": "Related speculative execution approach for agentic systems, complementary to Sherlock."
     },
     {
       "title": "Murakkab: Resource-efficient agentic workflow orchestration in cloud platforms",
       "authors": ["Ghufran I. Chaudhry", "Esha Choukse", "Haoran Qiu"],
       "year": 2025,
       "relevance": "Related work on agentic workflow serving infrastructure from overlapping author group."
     },
     {
       "title": "Circinus: Efficient query planner for compound ML serving",
       "authors": ["Banruo Liu", "Wei-Yu Lin", "Muhan Fang"],
       "year": 2025,
       "relevance": "SLO-aware query planner for compound AI workloads, complementary serving infrastructure."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark used in Sherlock's fault injection and evaluation experiments."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark used for Sherlock's end-to-end evaluation."
     }
   ]
 }