ai-research-survey

scan.json (28511B)

---

 {
   "paper": {
     "title": "Proof of Time: A Benchmark for Evaluating Scientific Idea Judgments",
     "authors": ["Bingyang Ye", "Shan Chen", "Jingxuan Tu", "Chen Liu", "Zidi Xiong", "Samuel Schmidgall", "Danielle S. Bitterman"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.07606",
     "doi": "10.48550/arXiv.2601.07606"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PoT introduces a time-partitioned benchmarking framework for evaluating scientific idea judgments across 30K+ instances in four domains (citations, awards, faculty research evolution, SOTA forecasting). Tool-using agents show large gains on evidence-exploration tasks (Faculty) but smaller or inconsistent gains on structured prediction tasks. Test-time compute scaling generally helps but varies by model family, and structured prompting is not a universal improvement. Post-cutoff evaluation materially shifts model rankings compared to pre-cutoff evaluation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link in the abstract: 'Code, data, and evaluation scripts are at https://github.com/shan23chen/proof_of_time'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states code, data, and evaluation scripts are released at the GitHub repository. The benchmark instances are described as JSONL files with a unified schema (Appendix A.8)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or specific library versions in the paper. The sandboxed Docker environment is mentioned for running agents but no dependency specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper references a GitHub repo but does not include a 'Reproducing Results' section or specific commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '35.1%' accuracy). Figure 2B shows error bars described as 'variability across task–model combinations' but the main tables (3, 7) report only point estimates without confidence intervals."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'Claude models show the steepest gains') but no statistical significance tests are reported. All comparisons are based on point estimate differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentage-point differences with baseline context, e.g., Table 3 shows pre-cutoff vs post-cutoff with deltas (+21.6pp, -17.7pp), and Table 7 shows scaling gains (Δ 15→50). These provide magnitude in context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Table 6 reports sample sizes per task (e.g., 200 for citations, 73 for professor field, 9 for field focus) but there is no justification for why these sizes were chosen or whether they provide adequate statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, confidence intervals, or variance across runs are reported. The paper does not mention running experiments multiple times or reporting spread measures. Single-run results appear to be the norm."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Zero-shot (direct generation) serves as the baseline against agentic configurations. Section 4.2 describes three solver configurations compared systematically."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models evaluated include frontier models from 2025-2026 (GPT-5.x, Claude 4.5, Gemini 2.5/3.x) as listed in Table 2. These are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper conducts controlled ablations over tool access (zero-shot vs agentic), structured prompting (vanilla vs structured prompt), and message-budget scaling (15/30/50), as described in Sections 4.2-4.3 and Figure 2."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only exact-match accuracy across all tasks (Section 4.1: 'We report exact-match accuracy for all tasks'). No secondary metrics such as calibration, rank correlation, or partial credit are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system outputs is performed. The LLM-as-judge analysis (Section 6) uses Gemini 3 pro, not human judges, to analyze agent traces."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses a temporal split: pre-cutoff data (2021-2024) as evidence, post-cutoff data (2025) as test. For Awards, 200 pre-cutoff papers are excluded from evidence to avoid leakage (Appendix A.3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task family (Citations, Awards, Faculty, SOTA) in Figure 2B, by model in Tables 3 and 7, and by model-task combination in Figure 7's heatmap."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides extensive failure analysis with a taxonomy (Table 4: reasoning errors 37.7%, retrieval/tooling 36.3%, etc.) and incomplete run analysis (Table 5). Table 10 provides bottleneck categories."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that structured prompting sometimes hurts (Section 5.3: GPT models 'often neutral or slightly worse'), that agents provide no benefit on SOTA tasks, and that post-cutoff evaluation can decrease scores (Table 3: Gemini 2.5 Flash -17.7pp)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about higher interaction budgets improving performance, task-dependent tool use benefits, and 30K+ instances are all supported by Tables 3, 7, Figures 2A-C, and Table 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses controlled ablations (tool access on/off, prompt variants, budget levels) with all other variables held constant. Section 4.2 describes the three solver configurations designed for controlled comparison. The causal claims about tool use and budget are supported by single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to NLP venues (ACL, NAACL, EMNLP), specific time windows, and acknowledges limitations of proxy targets, temporal choices, and offline sandbox realism in the Limitations section."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses proxy target imperfections (visibility, community dynamics), offline sandbox departure from real-world use, and the possibility that different agent architectures could change results. Section 5.4 notes pre-cutoff results may reflect both exposure and difficulty differences."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'We call PoT semi-verifiable because the benchmark uses verifiable downstream outcomes as imperfect proxies for idea quality. The signal is verifiable, but the target construct is not directly observable.' (Section 3). The Limitations section further discusses proxy limitations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 lists models as 'Claude Opus 4.5', 'GPT-5.2', 'Gemini 3 Pro Preview' etc. without specific API versions, snapshot dates, or model IDs. These are marketing names without version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The structured agent prompt is provided verbatim in Figure 14 (Appendix G). Task instances are described as JSONL with the user-facing prompt included (Appendix A.6-A.8)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states: 'All models were evaluated using their default API sampling parameters (temperature, top_p, etc.) without task-specific tuning.' Infrastructure settings are also specified (max retries: 3, timeout: 10 min for agent, 3 min for zero-shot, concurrent connections: 10)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 describes 'All agentic runs use a single-agent ReAct loop in a sandboxed environment.' Section 4.2 details the three solver configurations. The agent has 'access to bash, python, and a text editor.' The structured prompt is in Figure 14."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes data collection in detail: paper metadata from OpenReview (A.1), citation data from Google Scholar with temporal partitioning (A.2), award parsing (A.3), SOTA snapshot construction (A.4), professor publication history assembly with disambiguation heuristics (A.5)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the Conclusion, discussing proxy targets, temporal choices, offline sandbox realism, and agent loop constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section addresses specific threats: proxy target imperfections (citations influenced by visibility), the particular cutoff dates chosen affecting difficulty profiles, offline sandbox departing from real-world retrieval, and the single-agent ReAct loop potentially not generalizing to other architectures."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what it does not claim: 'Results should therefore be interpreted as measuring performance under a constrained evidence regime rather than full open-world assistance.' It also limits claims to the specific agent architecture and venues studied."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository promises code, data, and evaluation scripts. Appendix A.6 describes JSONL instance formats with exact sandbox manifests. The paper states 'We log full interaction traces, tool calls, and termination status' (Section 4.5)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes data collection in detail: OpenReview for paper metadata (A.1), Google Scholar for citations with specific temporal windows (A.2), award tiers from conference records (A.3), and SOTA from public leaderboards (A.4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public repositories (OpenReview, Google Scholar, public leaderboards)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A documents the pipeline: collection at t0, normalization into common schema, construction of sandbox manifests, collection at t1 for labels. Task instance format is documented (A.6-A.8). Table 6 provides final counts per task."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Harvard University, Mass General Brigham, Boston Children's Hospital, Brandeis University, Yale University, Johns Hopkins University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The paper evaluates commercial models (OpenAI, Google, Anthropic) — potential conflicts are not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states: 'we define t0 as January 2025 for all models whose knowledge cutoff precedes that date... The only exception is GPT-5.2, whose knowledge cutoff is August 31, 2025, so for GPT-5.2 we set t0 to September 1, 2025.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core design addresses this: Section 3.1 describes time-partitioned evaluation separating pre-cutoff evidence from post-cutoff targets. Section 5.4 (RQ4) explicitly evaluates contamination effects by comparing pre-cutoff vs post-cutoff performance."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination resistance is a central design principle. Section 3.2 describes the offline sandbox to 'minimize leakage through opportunistic retrieval.' The temporal partitioning design ensures post-cutoff labels were not available during model training. Section 2.2 discusses this in the context of related work."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports total API spend per model (e.g., GPT-5.2: $212.10, Claude Opus 4.5: $1781.51). Table 9 reports token usage per run across message budgets. Section 7 discusses cost-performance tradeoffs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports total API spend of $11,364.33 for all experiments. Table 9 provides detailed token usage. Appendix D provides cost-performance analysis."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of running experiments across multiple random seeds. Results appear to be from single runs per configuration."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many runs produced each result. No mention of 'averaged over K runs' or similar."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. The paper uses default API parameters (Appendix B), but the structured prompt design and message limits were presumably tuned without reporting a search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results for all configurations (zero-shot, agentic, agentic+structured prompt) at all budget levels (15/30/50) rather than selecting a best configuration. This comprehensive reporting avoids cherry-picking."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied. The paper makes many comparisons across models, tasks, and configurations without any correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the benchmark and evaluate models on it without discussing potential bias from benchmark design choices favoring certain model behaviors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 and Figures 11-12 explicitly analyze performance as a function of compute budget (token overhead vs accuracy gain). The message-budget scaling analysis (15/30/50) also addresses this."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses construct validity: 'We call PoT semi-verifiable because the benchmark uses verifiable downstream outcomes as imperfect proxies for idea quality' (Section 3). The Limitations section discusses proxy target imperfections."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models use the same agentic environment and tool interfaces: 'For agentic runs, the environment and tool interfaces are held constant across models' (Section 4.4). The scaffold is controlled across model comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is the core design consideration. Section 3.1 formalizes time-partitioned evaluation. The cutoff t0 is set before models' knowledge cutoffs to prevent temporal leakage. GPT-5.2 gets a different t0 due to its later cutoff (Appendix A)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The offline sandbox (Section 3.2) prevents feature leakage by mounting only pre-cutoff artifacts: 'Networking is disabled, and each task instance mounts a manifested set of read-only artifacts.' Post-cutoff labels are not accessible to solvers."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether pre-cutoff evidence papers and post-cutoff evaluation papers share structural similarities (same labs, same topics, near-duplicate methods) that could inflate performance."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The pre-cutoff vs post-cutoff comparison (Section 5.4, Table 3) serves as a contamination detection method, revealing which models may have been exposed to historical targets. The temporal split itself is a leakage prevention method."
       }
     }
   },
   "claims": [
     {
       "claim": "Higher interaction budgets generally improve agentic performance across most models.",
       "evidence": "Table 7 shows 15→50 message limit gains ranging from +12.1pp (GPT-5 nano) to +28.6pp (Gemini 3 Pro). Figure 2A visualizes these scaling curves.",
       "supported": "strong"
     },
     {
       "claim": "The benefit of tool use is strongly task dependent, with Faculty showing the largest improvement and SOTA showing minimal gain.",
       "evidence": "Figure 2B shows Faculty improving from near-zero (zero-shot) to ~two-thirds (agentic), Citations gaining ~10pp, Awards changing little, and SOTA remaining near ceiling in both settings.",
       "supported": "strong"
     },
     {
       "claim": "Structured agent prompts have family-dependent and model-dependent effects, not universal improvements.",
       "evidence": "Figure 2C shows Claude models tend to benefit, GPT models are neutral or slightly worse, and Gemini models show mixed outcomes (Section 5.3).",
       "supported": "moderate"
     },
     {
       "claim": "Post-cutoff evaluation materially shifts conclusions about model performance compared to pre-cutoff evaluation.",
       "evidence": "Table 3 shows shifts ranging from -17.7pp (Gemini 2.5 Flash) to +25.1pp (GPT-5.2) on Awards when moving from pre-cutoff to post-cutoff evaluation.",
       "supported": "strong"
     },
     {
       "claim": "Reasoning errors and retrieval/tooling failures are the dominant failure modes in agentic runs.",
       "evidence": "Table 4 shows reasoning errors at 37.7% and retrieval/tooling at 36.3% of complete-wrong runs. Table 5 shows looping/thrashing at 36.5% of incomplete runs (Section 6).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are single-run point estimates with no standard deviations, confidence intervals, or multiple-seed analysis. Given the stochastic nature of LLM outputs, single runs may not be reliable. The paper's own findings about 'lucky correctness' (Table 11: 43.1%) suggest results could be noisy."
     },
     {
       "flag": "Very small sample sizes for some tasks",
       "detail": "Field Focus has only 9 instances (Table 6). SOTA Bucket has 45 instances. Professor Field has 73. These small sizes make percentage-point comparisons unreliable without statistical tests."
     },
     {
       "flag": "LLM-as-judge for failure analysis without validation",
       "detail": "The failure taxonomy (Section 6) relies entirely on an LLM judge (Gemini 3 pro) without human validation of the judge's accuracy. The paper does not report inter-annotator agreement or judge reliability."
     },
     {
       "flag": "Single evaluation metric",
       "detail": "Only exact-match accuracy is reported. For tasks like citation bucket prediction or ranking, partial-credit metrics (e.g., rank correlation, bucket distance) could reveal different patterns but are not used."
     }
   ],
   "cited_papers": [
     {
       "title": "React: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2023,
       "relevance": "Core agentic framework (ReAct loop) used as the agent architecture in PoT's experimental setup."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Foundational work on LLM tool use, directly relevant to evaluating tool-using agents."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenges claims about emergent capabilities in LLMs, relevant to evaluating benchmark design and metric choice effects."
     },
     {
       "title": "Large language models cannot self-correct reasoning yet",
       "authors": ["Jie Huang", "Xinyun Chen", "Swaroop Mishra"],
       "year": 2023,
       "arxiv_id": "2310.01798",
       "relevance": "Relevant to understanding agent failure modes and the limits of self-correction in agentic reasoning loops."
     },
     {
       "title": "Benchmark data contamination of large language models: A survey",
       "authors": ["Cheng Xu", "Shuhao Guan", "Derek Greene", "M.-Tahar Kechadi"],
       "year": 2024,
       "arxiv_id": "2406.04244",
       "relevance": "Directly relevant survey on benchmark contamination, which PoT's temporal design aims to address."
     },
     {
       "title": "Livebench: A challenging, contamination-limited LLM benchmark",
       "authors": ["Colin White", "Samuel Dooley", "Manley Roberts"],
       "year": 2025,
       "arxiv_id": "2406.19314",
       "relevance": "Live benchmark approach to contamination resistance, parallel design philosophy to PoT's time-partitioned approach."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code evaluation benchmark using temporal partitioning, relevant to benchmark design methodology."
     },
     {
       "title": "Large language models are not fair evaluators",
       "authors": ["Peiyi Wang", "Lei Li", "Liang Chen"],
       "year": 2024,
       "relevance": "Documents inconsistencies in LLM-based judging, relevant to evaluating AI systems as scientific idea judges."
     },
     {
       "title": "Survey on evaluation of LLM-based agents",
       "authors": ["Asaf Yehudai", "Lilach Eden", "Alan Li"],
       "year": 2025,
       "arxiv_id": "2503.16416",
       "relevance": "Survey of agent evaluation methods including cost-efficiency and robustness, directly relevant to agentic evaluation methodology."
     },
     {
       "title": "Towards a science of scaling agent systems",
       "authors": ["Yubin Kim", "Ken Gu", "Chanwoo Park"],
       "year": 2025,
       "arxiv_id": "2512.08296",
       "relevance": "Framework for understanding when agents help vs. when they don't, directly relevant to PoT's agent evaluation findings."
     },
     {
       "title": "An information theoretic perspective on agentic system design",
       "authors": ["Shizhe He", "Avanika Narayan", "Ishan S. Khare"],
       "year": 2025,
       "arxiv_id": "2512.21720",
       "relevance": "Theoretical framework for agent design, relevant to understanding when agentic approaches provide value."
     },
     {
       "title": "ToolSandbox: A stateful, conversational, interactive evaluation benchmark for LLM tool use capabilities",
       "authors": ["Jiarui Lu", "Thomas Holleis", "Yizhe Zhang"],
       "year": 2025,
       "relevance": "Sandboxed tool-use evaluation benchmark, directly comparable to PoT's offline sandbox approach."
     }
   ]
 }