ai-research-survey

calibration.json (21704B)

---

 {
   "paper_slug": "adaptive-visionbased-coverage-2025",
   "calibrator": "opus",
   "calibration_date": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 46,
   "disagreement_count": 4,
   "agreement_rate": 0.92,
   "disagreements": [
     {
       "category": "statistical_methodology",
       "question": "effect_sizes_reported",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": true, "answer": true},
       "direction": "opus_generous",
       "explanation": "The schema states that 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' This paper reports '26.5% improvement in coverage' with raw numbers visible in Table 2 (91.8% vs baseline values), '32% reduction in energy consumption' (68 vs 85/94/100 units), and similar relative differences for all metrics. These are percentage improvements with baseline context, which the schema explicitly says qualifies as effect sizes. While the baselines chosen for percentage calculations are inconsistent (a separate concern), effect size magnitudes ARE reported."
     },
     {
       "category": "statistical_methodology",
       "question": "variance_reported",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "Sonnet credits the paper based on the statement that experiments are 'repeated five times with different random seeds' and that Table 2 includes 'Learning Stability (Variance).' However, the 'Learning Stability (Variance)' metric in Table 2 is reward variance within training (a performance metric comparing methods), NOT variance across the five experimental runs. The schema requires 'std dev in tables, variance across seeds, interquartile range, multiple-run results with spread measures.' The paper reports only averaged point estimates in Table 2 without any standard deviation or confidence intervals across the five runs. The mention that 'standard deviation of reward dropped below 5% of the mean after episode 100' describes within-run convergence behavior, not cross-run variability. No spread measure for the main results is provided."
     },
     {
       "category": "evaluation_design",
       "question": "ablation_study",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "Sonnet credits the reward coefficient sensitivity analysis (varying alpha, beta, gamma across 27 configurations) as an ablation study. However, this is a hyperparameter sensitivity analysis, not an ablation study. The schema asks: 'Is there an ablation study showing which components matter? Look for: experiments that remove or modify individual components to measure their contribution.' The paper's system has multiple distinct components (Dueling DQN architecture, Prioritized Experience Replay, target network, vision-based localization vs. alternatives, Double Q-learning). None of these are removed or isolated to measure their individual contribution. Sonnet itself acknowledges 'There is no ablation of architectural components.' Varying reward coefficients tests the sensitivity of a single design choice, not the contribution of system components."
     },
     {
       "category": "claims_and_evidence",
       "question": "abstract_claims_supported",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "The abstract claims '26.5% improvement in coverage' and '45% extension of network lifetime' 'compared to traditional distance-based localization.' The distance-based localization baseline is the non-vision MARL method (78.6% coverage). 91.8% vs 78.6% is a 16.8% improvement, not 26.5%. The 26.5% figure only matches if compared against static grid (72.5%), which is not distance-based localization. For network lifetime: 160 vs 120 (non-vision MARL) = 33%, not 45%. The 45% only matches against random placement (110 episodes), which is also not 'distance-based localization.' The abstract's claimed percentages are not supported by the results when the stated comparison baseline (distance-based localization) is used. Sonnet noted 'the baseline used for percentage calculation is not always made explicit' but still gave YES."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The implementation is described (Python, TensorFlow, OpenAI Gym-style interfaces) but no code is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The CNN was trained on a synthetic dataset of labeled sensor images, but this dataset is not made available. The simulation environment is custom and not shared."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python and TensorFlow but provides no version numbers, no requirements.txt, no Dockerfile, and no detailed environment specification. This is insufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or reproduction scripts are provided. While hyperparameters and architecture details are described, there are no instructions for running the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 2 and the text are reported as point estimates (e.g., 91.8% coverage, 68 energy units). No confidence intervals, error bars, or uncertainty ranges are provided despite averaging over 5 runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the proposed method 'consistently outperforms all baseline strategies' but performs no statistical significance tests. All comparisons are based on directly contrasting point estimates without p-values or any formal test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: '26.5% improvement in coverage' (with raw numbers 91.8% vs baselines in Table 2), '32% reduction in energy consumption' (68 vs 85/94/100 units), '22% decrease in redundancy,' and '45% extension of network lifetime.' The schema explicitly states that percentage improvement with baseline context counts as YES."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 100 sensor nodes, 200 training episodes, and 5 random seeds is not justified. No power analysis or argument is given for why these quantities are sufficient to draw reliable conclusions."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 2 are averaged over 5 runs but reported as point estimates only. The 'Learning Stability (Variance)' metric in Table 2 is reward variance within training (a performance metric), not variance across experimental runs. The statement about standard deviation dropping below 5% of the mean after episode 100 describes within-run convergence, not cross-run variability. No standard deviation, IQR, or spread measure for the main results across runs is provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against six methods: random placement, static grid deployment, centralized DRL (Liu & Wang 2021), RSSI-based MARL (Zhao & Chen 2022), graph-based MARL (Feng & Zhang 2023), and SLAM-DRL (Khan & Wu 2024). Multiple naive and state-of-the-art baselines are included."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The DRL/MARL baselines include work from 2021, 2022, 2023, and 2024, which are recent and represent current approaches. The paper also includes simpler baselines (random, static grid) for reference."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper performs a reward coefficient sensitivity analysis (varying alpha, beta, gamma across 27 configurations), but this is parameter tuning, not an ablation study. No system components are removed or isolated — the Dueling DQN architecture, Prioritized Experience Replay, target network stabilization, and vision-based localization are never individually ablated to measure their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports six metrics: final area coverage (%), energy consumption (units), redundancy rate (%), recovery time (seconds), network lifetime (episodes), and learning stability (variance). Multiple complementary metrics are used."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a sensor network coverage optimization paper evaluated entirely with automated simulation metrics. Human evaluation is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The system is trained and evaluated in the same simulation environment configuration (100 nodes, 500x500m, same parameters). There is no separate held-out test environment with different parameters used exclusively for final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as single aggregate numbers per method in Table 2. The scalability tests with 200 and 300 sensors are mentioned briefly in one sentence but not compared against baselines. No per-scenario, per-density, or per-condition breakdown is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss failure cases or conditions under which the proposed method underperforms. All reported results show the proposed method as uniformly superior across all metrics and conditions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows the proposed method as best. The reward sensitivity analysis notes that reducing beta leads to 'slightly lower total coverage' but this is presented positively as showing robustness. No configurations or conditions where the approach fails are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims '26.5% improvement in coverage' and '45% extension of network lifetime' 'compared to traditional distance-based localization.' The distance-based localization baseline (non-vision MARL using RSSI) achieves 78.6% coverage and 120 episodes lifetime. The actual improvements over this baseline are 16.8% (coverage) and 33% (lifetime), not 26.5% and 45%. The claimed percentages only match if cherry-picking different, weaker baselines for each metric (static grid for coverage, random placement for lifetime)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'incorporating real-time vision feedback into decentralized MARL significantly improves sensing efficiency' and that 'vision-based feedback eliminates the energy cost associated with inter-node localization messaging.' These are causal claims. The baselines differ in multiple ways simultaneously (localization method, architecture, communication), so the vision component's causal contribution cannot be isolated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "All experiments use a single 2D 500x500m simulation with 100 nodes, but the conclusion claims applicability to 'real-world MWSN deployments' in 'environmental monitoring and smart cities to disaster response and autonomous exploration.' The paper makes no attempt to bound these claims to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper does not consider whether results could stem from weak baseline implementations (described as 'simplified versions'), favorable hyperparameter tuning, or simulation artifacts. There is no threats-to-validity section."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'TensorFlow' without a version number. No version is given for Python or any other library used. While the RL architecture is described (Dueling Double DQN with 128/64/32 neurons), the software framework versions needed for reproduction are absent."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper uses deep reinforcement learning agents, not language model prompting. Prompts are not applicable."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV reports comprehensive hyperparameters: learning rate 1e-4, discount factor gamma=0.99, mini-batch size 64, epsilon decay from 1.0 to 0.1 over 200 episodes, replay buffer size 100,000, target network update every 1000 steps, reward coefficients alpha=1, beta=2, gamma=3, network architecture (128/64/32 neurons)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "This paper uses multi-agent reinforcement learning, not LLM-based agentic scaffolding. The MARL framework is described in detail but scaffolding in the LLM sense does not apply."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The vision processing pipeline is described in Section III: Gaussian blurring, adaptive thresholding, bounding box extraction, geometric calibration via fiducial markers, histogram equalization, and temporal filtering. The CNN architecture (3 conv layers with 3x3 kernels, ReLU, max pooling, 2 FC layers) is also detailed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper proceeds directly from performance evaluation to the conclusion section, which discusses only future work directions without acknowledging current limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. The brief mention that 'exact hardware-level power profiling was not performed' is the closest, but this is buried in the energy modeling discussion, not presented as a limitation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes no explicit statements about what its results do NOT show. The conclusion claims broad applicability to 'environmental monitoring and smart cities to disaster response and autonomous exploration' without bounding to the tested 2D simulation setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw simulation logs, per-run data, episode-level trajectories, or experimental data are made available. Only aggregated results in Table 2 and figures are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The simulation setup is described in detail: 100 mobile nodes in 500x500m area, sensing radius 20m, communication range 50m, battery energy 100 units, energy cost k=0.5 units/meter, 5 runs with different random seeds, 200 episodes each with 10-second windows."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a simulation study with synthetic sensor networks, so participant recruitment is not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The processing pipeline is documented: camera captures images at 10-second intervals, CNN detects and localizes LED positions (95.2% IoU accuracy), sensor positions are used to compute coverage metrics, metrics are translated into reward signals for the RL framework. The flow from data collection to analysis is traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section, no funding sources, no grants, and no financial support are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are listed with their affiliation: School of Electrical Engineering, Iran University of Science and Technology, Tehran, Iran."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding source is disclosed. The work appears to be unfunded academic research from a university. The schema says NA if unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement, no patent disclosure, and no declaration of financial interests anywhere in the paper. Absence of disclosure does not satisfy this criterion."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper trains a DRL agent from scratch in a custom simulation environment. No pre-trained language model with a training data cutoff is evaluated on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model with potential training data contamination is used. The DRL agent is trained online in the simulation environment."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on an existing benchmark. The evaluation uses a custom simulation, so benchmark contamination does not apply."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a simulation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Inclusion/exclusion criteria are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Randomization of participants is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Attrition reporting is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses computational complexity in O-notation (Section III.E) but reports no actual wall-clock inference time, per-decision latency, GPU/CPU requirements, or cost figures for running the system."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No hardware specifications, GPU hours, training time, or total computational budget are stated. The paper mentions TensorFlow but provides no information about the hardware used or time required for training."
       }
     }
   }
 }