ai-research-survey

calibration.json (17500B)

---

 {
   "calibration": {
     "paper_slug": "agentsllm-augmentative-generation-2025",
     "calibration_date": "2026-02-28",
     "sonnet_scan_date": "2026-02-28",
     "agreement_rate": 0.98,
     "total_questions": 50,
     "agreements": 49,
     "disagreements": 1,
     "disagreement_details": [
       {
         "category": "setup_transparency",
         "question": "prompts_provided",
         "sonnet": { "applies": true, "answer": true },
         "opus": { "applies": true, "answer": false },
         "direction": "sonnet_generous",
         "explanation": "Sonnet credits Figure 2 as providing sufficient prompt text for the SMA. Opus notes that only the SMA prompt is shown in full; the QA agents' prompts (Text QA Agent, Visual QA Engineer, Visual QA Agent) are described only in natural language without verbatim text. The schema requires that 'the reader must be able to reconstruct every prompt sent to the model,' which is not possible for the tQA and vQA variants. Additionally, the 'list of common problems compiled from typical mistakes observed during initial experimentation' given to the Text QA Agent is referenced but never provided. Opus answer: false."
       }
     ],
     "opus_checklist": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "The conclusion states 'The code for this paper will be made available,' which is a promise of future release. A GitHub link is in the header but the conclusion confirms code is not yet available. Per schema, a promise counts as NO."
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses publicly available interPlan scenarios [9] and the nuPlan benchmark [2]. These are standard public benchmarks the authors did not modify. No proprietary data was collected."
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, conda environment file, or library version listing is provided. Only LLM model names are mentioned without software dependency details."
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method description is high-level and insufficient for a competent researcher to reproduce without guessing."
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Table IV reports 95% confidence intervals via bootstrapping for Elo ratings for all nine variants (e.g., interPlan: 1042, CI -9/+11). CIs are not provided for displacement error, but the Elo CIs satisfy the criterion."
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No formal significance tests (p-values, t-tests, etc.) are used. Claims like 'significantly larger error' for LCTGen and 'significantly fewer errors' for GPT-4o are made without statistical testing. The Elo CI-based ranking is not a formal hypothesis test."
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": false,
           "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are computed. Raw displacement errors and Elo point differences are reported but without standardized effect size measures. Table V provides driving scores with baseline context but no formal effect size metric."
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 50-scenario test set and 9 expert judges are not justified with power analysis or explicit reasoning. No acknowledgment that these numbers may be insufficient for certain claims."
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Figures 3 and 4 show box plots (distributions across scenarios) but these are not variance across experimental runs — each configuration appears to be run once per scenario. Table V reports only mean driving scores with no variance. No standard deviation or IQR is reported in tabular form."
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "LCTGen [23] serves as an external baseline for displacement error (Figure 3). Human-generated interPlan scenarios serve as ground truth in the Elo comparison (Table IV)."
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "LCTGen (CoRL 2023) and interPlan (IROS 2024) are both recent and appropriate for the task. No suspiciously old or weak baselines."
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Table II and Section V.B present four variants (OTM, FC, tQA, vQA) that systematically vary components: lane representation and QA loop type. This constitutes an ablation over framework components."
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Three metrics: (1) displacement error in meters, (2) Elo rating from human pairwise comparisons, (3) nuPlan mean driving score."
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Nine domain experts conducted 5760 pairwise comparisons of BEV images in a blind setup with model identities hidden. Elo ratings with 95% CIs computed."
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "The same 50 interPlan scenarios are used for both development ('ground-truth for development') and final evaluation. No held-out split is described."
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Figure 3 shows displacement error by five scenario types. Table III shows error counts by three error categories (position, heading, logic) per model."
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section V.A defines three error categories with Table III quantifying them. Figure 5 provides qualitative failure examples (e.g., 'Gemini placed the accident vehicles with unrealistically large overlap')."
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Multiple negative results: vQA does not improve displacement error over tQA (Section V.B); smaller models significantly underperform with OTM; LCTGen fails entirely on two scenario types."
         }
       },
       "claims_and_evidence": {
         "abstract_claims_supported": {
           "applies": true,
           "answer": true,
           "justification": "Abstract claims about comparable quality to manual scenarios are supported by Table IV (GPT-4o OTM Elo 1039 vs. interPlan 1042, overlapping CIs, both rank 1). The claim about smaller models is supported by Gemini vQA reaching rank 1."
         },
         "causal_claims_justified": {
           "applies": true,
           "answer": true,
           "justification": "Causal claims about the agentic framework enabling smaller models are supported by controlled ablation experiments (Table II, Figures 3-4) that vary prompting strategy while holding the model constant, allowing causal attribution."
         },
         "generalization_bounded": {
           "applies": true,
           "answer": false,
           "justification": "Tested on only 50 scenarios from one dataset (interPlan) covering five types on nuPlan. The title and abstract make broad claims about 'autonomous driving planners' and 'LLM-agent based framework' without bounding results to this specific setting."
         },
         "alternative_explanations_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No alternative explanations are discussed. The discrepancy between vQA improving Elo but not displacement error is not explained (could be evaluator visual bias). No confounds or robustness checks are discussed."
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Uses 'GPT-4o', 'Gemini-1.5-Flash', 'Llama3.1-70B' without API version numbers or snapshot dates. Per schema, marketing names without snapshot dates do not count."
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "Figure 2 shows the SMA prompt with actual values for one example, but the QA agents' prompts (Text QA, Visual QA Engineer, Visual QA Agent) are only described in natural language. The 'list of common problems' given to the Text QA Agent is referenced but not provided. The reader cannot reconstruct all prompts used across tQA and vQA variants."
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "No LLM hyperparameters (temperature, top-p, max tokens) reported. The paper states models are used 'without any problem-specific fine-tuning' but does not report inference sampling settings."
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Section III describes the agentic scaffold in detail: SMA, Text QA Agent, Visual QA Engineer, Visual QA Agent with their roles, inputs, outputs, rating mechanism (1-5 scale), feedback loops, and retry logic. Figure 1 provides a workflow diagram."
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section III.C and Table I document the text-based scenario representation (entity types, attributes, vector format). The transformation from raw interPlan scenarios to input vectors is documented."
         }
       },
       "limitations_and_scope": {
         "limitations_section_present": {
           "applies": true,
           "answer": false,
           "justification": "No dedicated limitations section. One sentence in the conclusion: 'One limitation is the dependence on commercial frontier LLMs.' Per schema, a single sentence in the conclusion does not count as substantive discussion."
         },
         "threats_to_validity_specific": {
           "applies": true,
           "answer": false,
           "justification": "No threats to validity discussed. The one mentioned limitation (API dependence) is a practical concern, not a validity threat. Specific threats (small test set, evaluator bias, no held-out split) are not discussed."
         },
         "scope_boundaries_stated": {
           "applies": true,
           "answer": false,
           "justification": "No explicit out-of-scope statements. Results are not bounded to the five interPlan scenario types, the nuPlan map format, or the specific LLMs tested."
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Raw generated scenarios and expert judgment data (5760 pairwise comparisons) are not released. The underlying data cannot be independently verified."
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section IV describes the 50 interPlan scenarios across five types, how they are used for development and evaluation, and how expert pairwise comparisons were collected."
         },
         "recruitment_methods_described": {
           "applies": true,
           "answer": false,
           "justification": "Nine expert judges described only as 'experts from the autonomous driving research community.' No description of how they were recruited, what criteria defined 'expert,' or whether they were internal colleagues or external."
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline is documented: interPlan scenarios → text-based representation → LLM modification → optional QA loop → nuPlan simulation → evaluation. Pairwise comparison → Elo computation is also described."
         }
       },
       "conflicts_of_interest": {
         "funding_disclosed": {
           "applies": true,
           "answer": false,
           "justification": "No acknowledgments section or funding disclosure. Multiple authors are from Robert Bosch GmbH, a major industry player, but no explicit funding statement is made."
         },
         "affiliations_disclosed": {
           "applies": true,
           "answer": true,
           "justification": "Author affiliations clearly listed: Robert Bosch GmbH, FAU Erlangen-Nürnberg, University of Toronto, ScaDS.AI/TU Dresden, University of Tübingen, Vector Institute."
         },
         "funder_independent_of_outcome": {
           "applies": true,
           "answer": false,
           "justification": "Multiple authors affiliated with Robert Bosch GmbH, which has commercial interests in autonomous driving testing. The employer has a direct stake in demonstrating the framework's effectiveness."
         },
         "financial_interests_declared": {
           "applies": true,
           "answer": false,
           "justification": "No competing interests statement or financial interest declaration is present in the paper."
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "The paper uses LLMs for generative scenario creation (instruction-following), not for benchmark knowledge retrieval. The LLMs are not being evaluated on what they know from training data. Contamination is structurally inapplicable."
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable — the task is generative scenario augmentation from user instructions, not benchmark knowledge retrieval. No meaningful train/test contamination concern."
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable — the paper does not evaluate pre-trained model knowledge on a benchmark. nuPlan and interPlan are used to evaluate driving planners, not LLM knowledge."
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": true,
           "answer": false,
           "justification": "No pre-registration link provided (no OSF, AsPredicted, or other registry). The human expert evaluation study is not pre-registered."
         },
         "irb_or_ethics_approval": {
           "applies": true,
           "answer": false,
           "justification": "No mention of IRB or ethics board approval despite involving nine human participants in an evaluation study."
         },
         "demographics_reported": {
           "applies": true,
           "answer": false,
           "justification": "Expert judges described only as 'experts from the autonomous driving research community.' No demographics (experience level, years, gender, geographic distribution) reported."
         },
         "inclusion_exclusion_criteria": {
           "applies": true,
           "answer": false,
           "justification": "No inclusion/exclusion criteria for expert judges stated beyond 'experts from the autonomous driving research community.' No formal screening or expertise definition."
         },
         "randomization_described": {
           "applies": true,
           "answer": true,
           "justification": "Section IV states 'The order of match-ups were randomized,' indicating randomization of pairwise comparison order to prevent ordering effects."
         },
         "blinding_described": {
           "applies": true,
           "answer": true,
           "justification": "Section IV states 'the identities of the models were hidden from the judges during rating,' confirming evaluators were blinded to model identity."
         },
         "attrition_reported": {
           "applies": true,
           "answer": false,
           "justification": "No attrition information. 5760 comparisons across 9 judges stated but no per-judge breakdown or dropout information provided."
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Cost is discussed qualitatively ('very cost inefficient' for vQA with frontier models) but no actual figures (API spend, token counts, cost per scenario) are reported despite cost reduction being a stated motivation."
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget stated. No GPU hours, API spend, or hardware specifications provided."
         }
       }
     }
   }
 }