ai-research-survey

scan.json (25445B)

---

 {
   "paper": {
     "title": "ResearchRubrics: A Benchmark of Prompts and Rubrics For Evaluating Deep Research Agents",
     "authors": ["Manasi Sharma", "Chen Bo Calvin Zhang", "Chaithanya Bandi", "Clinton Wang", "Ankit Aich", "Huy Nghiem", "Tahseen Rabbani", "Ye Htet", "Brian Jang", "Sumana Basu", "Aishwarya Balwani", "Denis Peskoff", "Marcos Ayestaran", "Sean M. Hendryx", "Brad Kenstler", "Bing Liu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.07685",
     "doi": "10.48550/arXiv.2511.07685"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RESEARCHRUBRICS introduces 101 prompts with 2,593 expert-written rubric criteria for evaluating Deep Research agents across 9 domains. Even leading agents (Gemini DR, OpenAI DR) achieve under 68% rubric compliance. Implicit reasoning and synthesis account for 45-50% of all failures. LLM-based rubric augmentation catastrophically degrades human-LLM alignment by 15-20%, while adding concrete examples improves it by 3-4%.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'we release RESEARCHRUBRICS (including all prompts, rubrics, and evaluation code)' and provides a URL: https://scale.com/research/researchrubrics."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark includes all prompts and rubrics as released data. The abstract confirms release of all prompts and rubrics."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency information is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The evaluation prompts are shown (Figs. 19-21) but there are no instructions for replicating the full evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 5 report point estimates (e.g., 0.677, 0.664) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Gemini DR outperforms OpenAI DR and Perplexity DR but provides no statistical significance tests for these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute compliance scores and percentage breakdowns (e.g., 67.7% ternary, 61.5% binary) with baseline context, and failure rate breakdowns by category. Correlation coefficients with p-values are reported for length-quality analysis (e.g., r=0.280, p=0.005)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 101 prompts but no justification is given for why this number is sufficient. No power analysis or sample size discussion."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for the main compliance scores. Table 9 reports SD for response lengths but not for evaluation scores."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three commercial DR systems are compared against each other (Gemini DR, OpenAI DR, Perplexity DR), and results are contextualized against other benchmarks (LiveResearchBench, DeepResearch Bench)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All three evaluated systems (OpenAI DR, Gemini DR, Perplexity DR) are state-of-the-art commercial deep research agents as of 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 presents ablation studies on rubric design: example detail (low vs. high) and LLM augmentation (absent vs. present), shown in Table 7."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple evaluation schemes are used: binary vs. ternary grading, Macro F1 for human-LLM alignment, per-category failure rates, mandatory vs. optional criteria breakdowns."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Nine expert annotators provided ground truth across 303 responses for human-LLM judge alignment analysis (Section 4.3, Table 6)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper evaluating commercial systems, not a training/tuning study. There is no train/test split to hold out."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: by rubric axis (Fig. 5), by domain (Fig. 14), by complexity dimension (Fig. 6), by mandatory vs. optional criteria (Fig. 8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses failure patterns in detail: implicit reasoning and synthesis failures (45-50%), mandatory vs. optional criteria failures, breadth-accuracy trade-off in citations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The LLM augmentation ablation (Table 7) shows a clearly negative result: automated rubric expansion 'catastrophically degrades alignment by 15-20%'."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims ('under 68% average compliance', 'missed implicit context and inadequate reasoning') are supported by Table 5 (67.7% max) and Fig. 5 (implicit criteria dominant failure mode)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'improving Deep Research agents requires architectural innovation rather than incremental refinement' (Conclusion) and 'current architectures inherently favor creative synthesis over systematic execution' (Section 4.5) without adequate evidence for these causal/architectural claims. These are speculative explanations for correlational observations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests 3 commercial DR systems on 101 prompts but makes broad claims about 'fundamental architectural limitations' and 'current systems' generally. The title and claims extend well beyond the three specific systems tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses the length-quality conflation and considers whether correlation reflects 'genuine informational density rather than stylistic inflation.' Section 4.5 discusses multiple explanations for observed patterns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit that rubric compliance is the proxy and discusses its relationship to actual research quality. The distinction between mandatory and optional criteria addresses sufficiency vs. excellence. Section 4.2 discusses the length-quality conflation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "LLM judges are listed as 'GPT-5', 'Claude-Sonnet-4.5', 'Gemini-2.5-Pro' without specific version snapshots or API dates. The evaluated DR systems are named by product (e.g., 'OpenAI Deep Research') without versioning."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full evaluation prompts are provided in Figures 19-21, including the system prompt, user prompt, example removal prompt, and augmentation prompt."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for either the LLM judges or the DR systems being evaluated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party commercial DR tools (OpenAI DR, Gemini DR, Perplexity DR) as black boxes. The authors cannot describe their internal scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage data collection pipeline is described in detail (Fig. 2, Section 3.1): Expert 1 proposes, Expert 2 reviews and iterates, Expert 3 makes final adjustments. PDF-to-markdown conversion of responses is mentioned in Section 4.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Conclusion mentions 'Future Work' but does not substantively discuss limitations of the benchmark itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address threats like annotator bias, prompt selection bias, or temporal validity of results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to the 3 tested systems or discuss populations/settings excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases all prompts, rubrics, and evaluation code. The raw human annotation data for the 303 responses appears available through the release."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the three-expert pipeline, the source of prompt ideas (user forums, Q&A sites, brainstorming), and the iterative refinement process in detail (Fig. 2)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '9 expert annotators' for human evaluation and 'experts' for data collection but does not describe how these experts were recruited, their qualifications beyond 'strong STEM background', or potential selection biases."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from prompt creation (3-stage expert process) through evaluation (LLM-as-judge with ternary scoring) to final score calculation (Equation 1) is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. Most authors are affiliated with Scale AI (a commercial company) but no funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Scale AI (majority of authors), University of Maryland, University of Chicago, Washington University, McGill University, UC Berkeley."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Scale AI is a data labeling and evaluation company that sells benchmark and evaluation services. The majority of authors are Scale AI employees. Scale AI has a direct commercial interest in demonstrating the need for human-expert evaluation benchmarks, which is exactly what this paper advocates."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest declarations are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the LLM judges (GPT-5, Claude-Sonnet-4.5, Gemini-2.5-Pro) or the evaluated DR systems."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark prompts or rubrics could appear in LLM training data. Since the rubrics are expert-written and novel, this risk is lower but is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any benchmark content was available online before the models' training cutoffs. The paper does not address contamination for either the evaluated systems or the LLM judges."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The paper includes human annotators (9 experts for 303 responses). No pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human evaluation study."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The 9 expert annotators are not characterized beyond being 'experts'. No demographics, experience levels, or background information provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion/exclusion criteria for annotator selection are described. Experts are defined as 'individual with a strong STEM background who is skilled in task design and evaluation' but no formal criteria are given."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment conditions requiring randomization. It is a benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not clearly applicable to this benchmark evaluation design."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information about whether all 9 annotators completed all assigned tasks or if there was any dropout."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs are reported for running the DR systems or the LLM judges across the 101 prompts."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper mentions '2,800+ hours of human labor' for benchmark creation but not compute costs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results appear to be from single runs of each DR system. LLM judge results also appear unreplicated."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs per system is not stated. It appears each DR system was run once per prompt."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates commercial systems with no hyperparameter tuning. Not applicable."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection is involved — commercial systems are used as-is."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 3 systems, multiple domains, complexity dimensions, and rubric categories with no correction applied."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not propose its own system — it evaluates third-party systems. Self-comparison bias is not applicable."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of compute budgets across systems. Perplexity produces much shorter outputs (suggesting less compute) but this is not analyzed as a compute variable."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 directly addresses construct validity of rubric-based evaluation through ablation studies on rubric design. The human consistency analysis (Table 6) validates the benchmark against human judgment."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates bundled commercial DR products as products, not isolating model vs. scaffold contributions. NA since the tools are the thing being tested."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether DR systems may have encountered similar prompts or rubric patterns during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The LLM judges are given full rubrics — no analysis of whether this introduces bias."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between the benchmark and training data of evaluated systems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is employed."
       }
     }
   },
   "claims": [
     {
       "claim": "Even leading DR agents achieve under 68% average rubric compliance",
       "evidence": "Table 5: Gemini DR 67.7% ternary, 61.5% binary; OpenAI DR 66.4% ternary, 59.7% binary; Perplexity DR 56.6% ternary, 48.7% binary",
       "supported": "strong"
     },
     {
       "claim": "Implicit reasoning and synthesis jointly account for 45-50% of all failures",
       "evidence": "Fig. 5 shows implicit criteria failure rates of ~49% and synthesis ~25-29% across all systems",
       "supported": "strong"
     },
     {
       "claim": "Binary grading achieves substantially higher human-LLM agreement than ternary (0.72-0.76 vs 0.53-0.57 Macro F1)",
       "evidence": "Table 6: binary F1 ranges 0.717-0.760, ternary ranges 0.527-0.567",
       "supported": "strong"
     },
     {
       "claim": "LLM-based rubric augmentation catastrophically degrades alignment by 15-20%",
       "evidence": "Table 7: augmentation drops binary F1 from 0.721-0.760 to 0.508-0.564",
       "supported": "strong"
     },
     {
       "claim": "Performance degrades monotonically with increased logical nesting depth",
       "evidence": "Fig. 6/13: All three systems show decreasing scores from shallow to deep nesting, though the decrease is modest (~3-5pp for most systems)",
       "supported": "moderate"
     },
     {
       "claim": "Systematic failures indicate fundamental architectural limitations rather than implementation differences",
       "evidence": "Section 4.5 argues this from consistency of failure patterns, but this is an interpretive claim without direct architectural evidence",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating commercial space it serves",
       "detail": "Scale AI is a data labeling and evaluation company. The paper's core argument — that human-expert evaluation benchmarks are needed and LLM-generated rubrics are insufficient — directly aligns with Scale AI's commercial interests. 11 of 16 authors are Scale AI employees."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section. For a benchmark paper making strong claims about 'fundamental architectural limitations' of DR systems, the absence of self-critical discussion of the benchmark's own limitations is notable."
     },
     {
       "flag": "Overclaiming from limited sample",
       "detail": "Claims about 'fundamental architectural constraints' and 'current systems' are based on testing only 3 commercial DR systems on 101 prompts. The leap from correlational findings to architectural conclusions is not justified."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Performance differences between systems are presented without any statistical testing, yet comparative rankings are prominently discussed."
     },
     {
       "flag": "Single-run evaluation",
       "detail": "DR systems appear to have been run only once per prompt. Given the stochastic nature of LLM outputs, single-run results may not be stable. No variance across runs is reported."
     }
   ],
   "cited_papers": [
     {
       "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health",
       "authors": ["R. K. Arora", "J. Wei"],
       "year": 2025,
       "arxiv_id": "2505.08775",
       "relevance": "Related rubric-based LLM evaluation benchmark; comparison point for human-LLM judge alignment methodology."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["M. Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Taxonomy of multi-agent LLM system failures (MAST); cited for reasoning-action mismatch rates in agent systems."
     },
     {
       "title": "Limits to Scalable Evaluation at the Frontier: LLM as Judge Won't Beat Twice the Data",
       "authors": ["F. E. Dorner", "V. Y. Nastl", "M. Hardt"],
       "year": 2025,
       "arxiv_id": "2410.13341",
       "relevance": "Challenges assumptions about LLM-as-judge scalability; raises concerns about circularity in LLM-generated evaluation."
     },
     {
       "title": "Towards an AI Co-Scientist",
       "authors": ["J. Gottweis"],
       "year": 2025,
       "arxiv_id": "2502.18864",
       "relevance": "AI co-scientist built on multi-agent Gemini 2.0; raises stakes for evaluating AI research capabilities."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "authors": ["G. Mialon"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "General AI assistant benchmark; comparison point for DR evaluation methodology."
     },
     {
       "title": "Humanity's Last Exam",
       "authors": ["L. Phan"],
       "year": 2025,
       "arxiv_id": "2501.14249",
       "relevance": "Expert-level evaluation benchmark; DR systems scored 26.6% showing capability gaps."
     },
     {
       "title": "ExpertLongBench: Benchmarking Language Models on Expert-Level Long-Form Generation Tasks with Structured Checklists",
       "authors": ["J. Ruan"],
       "year": 2025,
       "arxiv_id": "2506.01241",
       "relevance": "Related benchmark using structured checklists for long-form generation evaluation; close comparison point."
     },
     {
       "title": "Mind2Web 2: Evaluating Agentic Search with Agent-as-a-Judge",
       "authors": ["B. Gou"],
       "year": 2025,
       "relevance": "Agent-as-a-Judge evaluation framework for agentic search; related evaluation methodology."
     },
     {
       "title": "HotpotQA: A Dataset for Diverse, Explainable Multi-hop Question Answering",
       "authors": ["Z. Yang"],
       "year": 2018,
       "doi": "10.18653/v1/D18-1259",
       "relevance": "Multi-hop QA benchmark; cited for limitations of multi-hop reasoning in current agent architectures."
     },
     {
       "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents",
       "authors": ["J. Wei"],
       "year": 2025,
       "arxiv_id": "2504.12516",
       "relevance": "Benchmark for browsing agents; comparison point in the DR benchmark landscape."
     }
   ]
 }