ai-research-survey

calibration.json (20112B)

---

 {
   "paper_slug": "3dshape2vecset-3d-shape-2023",
   "total_questions": 50,
   "agreement_count": 46,
   "disagreement_count": 4,
   "agreement_rate": 0.92,
   "disagreements": [
     {
       "category": "statistical_methodology",
       "question": "effect_sizes_reported",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "The paper reports raw metric values with baseline context throughout. For example, Table 3 shows IoU improvements from 0.953 (3DILG) to 0.965 (Ours), Table 6 shows Surface-FPD from 1.89 (3DILG) to 0.76 (Ours). Per the schema, 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper consistently provides baseline values alongside its own, enabling the reader to assess the magnitude of improvement.",
       "sonnet_justification": "Raw metric differences are shown in tables but no standardized effect sizes (Cohen's d, etc.) are reported. The paper does not provide baseline context for interpreting magnitude of improvements.",
       "direction": "opus_generous"
     },
     {
       "category": "evaluation_design",
       "question": "human_evaluation",
       "sonnet_applies": false,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "Human evaluation is relevant to 3D shape generation quality claims. Automated metrics like FID and IoU are proxies; human perceptual judgment of generated 3D shapes is a meaningful evaluation that the paper could have but did not include. The schema says 'NA if human evaluation is clearly irrelevant to the claims' -- but for generative modeling, human evaluation of output quality is standard and relevant. The paper did not include any human evaluation.",
       "sonnet_justification": "This is a 3D shape reconstruction and generation paper evaluated entirely with automated metrics (IoU, FID, etc.); human evaluation is not relevant to the geometric quality claims being made.",
       "direction": "applies_boundary"
     },
     {
       "category": "evaluation_design",
       "question": "failure_cases_discussed",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "Section 4 explicitly states: 'We initially explored many variations for 3D shape representation based on irregular and regular grids as well as tri-planes, frequency compositions, and other factored representations. Ultimately, we could not improve on existing irregular grids.' This constitutes a discussion of where certain approaches failed. Additionally, Section 8.8 (Limitations) discusses the drawbacks of the two-stage training strategy, including the cost of retraining. While not a traditional failure-case visualization, the paper does discuss where the approach has weaknesses.",
       "sonnet_justification": "The limitations section (8.8) discusses training cost issues but does not show failure cases or discuss specific shapes/scenarios where the method breaks down. Only successes are visualized.",
       "direction": "opus_generous"
     },
     {
       "category": "cost_and_practicality",
       "question": "compute_budget_stated",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "Section 7.3 states: 'The shape auto-encoder is trained on 8 A100, with batch size of 512 for T=1,600 epochs' and 'The diffusion models are trained on 4 A100 with batch size of 256 for T=8,000 epochs.' While the paper does not give a single total GPU-hours figure, it specifies the hardware (8 A100 GPUs, 4 A100 GPUs), the number of epochs (1,600 and 8,000), and batch sizes. This is sufficient for a reader to understand the approximate compute budget. The schema asks 'Is the total computational budget stated? Look for: GPU hours, total API spend, hardware used, training time.' Hardware used and training time (in epochs) are provided.",
       "sonnet_justification": "While hardware (8 A100 for autoencoder, 4 A100 for diffusion) and epoch counts are stated, the total GPU-hours or compute budget is not explicitly quantified.",
       "direction": "opus_generous"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code: https://1zb.github.io/3DShape2VecSet/' -- a working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses ShapeNet-v2 (Chang et al. 2015), a publicly available standard benchmark dataset. Per schema guidance, 'If the data is a standard public benchmark they didn't modify, YES.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes hardware (8 A100 GPUs for autoencoder, 4 A100 for diffusion) but provides no requirements.txt, Dockerfile, conda file, or library version specifications. Only 'Python' is implied without detailed dependency information."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "A code URL is provided but the paper text itself does not include step-by-step reproduction instructions or commands to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3-9 report only point estimates (IoU, Chamfer, F-Score, FID, KID values) with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'we can beat PVD by a large margin') without any statistical significance tests. No p-values, t-tests, or bootstrap tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Per the schema definition, reporting baseline values alongside the method's own results provides the context needed. Tables 3-9 consistently show baseline metrics next to the proposed method's metrics (e.g., IoU 0.953 for 3DILG vs 0.965 for Ours in Table 3; FPD 1.89 for 3DILG vs 0.76 for Ours in Table 6). The reader can directly assess the magnitude of improvement from these comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for why the number of test shapes or categories is sufficient for the comparative claims. The evaluation uses ShapeNet-v2 with 55 categories but no justification for adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviations, variance across seeds, or multiple-run results are reported in any table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against OccNet, ConvOccNet, IF-Net, 3DILG for shape autoencoding (Table 3); PVD, 3DILG, NeuralWavelet, Grid-83 for generation (Tables 6-9); AutoSDF for text-conditioned; ShapeFormer for completion."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include 3DILG (NeurIPS 2022), NeuralWavelet (SIGGRAPH Asia 2022), PVD (ICCV 2021) -- all contemporary to this 2023 paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 ablates number of latents M (512, 256, 128, 64). Table 5 ablates KL channel size C0 (1, 2, 4, 8, 16, 32, 64). Section 8.1 compares Learned Queries vs. Point Queries. Table 6 shows generation results across different C0 values."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Shape autoencoding uses IoU, Chamfer distance, and F-Score. Generation uses Surface-FPD, Surface-KPD, Rendering-FID, Rendering-KID. Category-conditioned adds Precision, Recall, MMD-CD, MMD-EMD, COV-CD, COV-EMD."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "This is a generative modeling paper. Human perceptual evaluation of generated 3D shapes is a relevant and commonly used evaluation method in generative modeling research. The paper relies entirely on automated metrics (FID, KID, IoU, etc.) with no human evaluation of output quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states 'We use the training/val splits in [Zhang et al. 2022]'. Section 8.1 refers to 'test split' shapes for visualization. Explicit training/test separation is used."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-category results for the 7 largest ShapeNet categories plus averages. Table 8 shows per-category generation results for airplane, chair, table, car, sofa. Table 9 adds chair and table breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'We initially explored many variations for 3D shape representation based on irregular and regular grids as well as tri-planes, frequency compositions, and other factored representations. Ultimately, we could not improve on existing irregular grids.' This is an explicit discussion of approaches that failed. Section 8.8 discusses limitations of the two-stage training approach."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 reports that C0=64 gives worse generation results than C0=32. Section 4 discusses that irregular/regular grids and tri-planes were explored but 'could not improve on existing irregular grids.' These are genuine negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'improved performance in 3D shape encoding and 3D shape generative modeling tasks' -- supported by Tables 3-9 showing improvements over baselines. It claims a 'wide variety of generative applications' -- supported by Sections 8.2-8.6 covering unconditional, category-conditioned, text-conditioned, completion, and image-conditioned generation."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (e.g., 'Point Queries are generally better than Learned Queries', effect of M and C0). These are supported by controlled single-variable ablation experiments in Tables 3-5 where one component is varied while others are held constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'improved performance in 3D shape encoding and 3D shape generative modeling tasks' without bounding to ShapeNet. The title is broadly framed. All experiments use only ShapeNet-v2 (synthetic man-made objects). No explicit scope boundaries are stated regarding the data domain."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for observed improvements. It does not consider whether gains come from the representation itself, increased model capacity, the attention architecture, or the specific training recipe. The limitations section only discusses training cost, not confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper trains its own neural networks from scratch. It does not use pre-trained LLMs or API-based models. BERT is used for text conditioning but referenced by citation, and the schema question targets LLM API versions which is not applicable here."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting or LLM APIs. All models are trained end-to-end from scratch. Text conditioning uses BERT embeddings, not prompt-based generation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 provides detailed hyperparameters: batch sizes (512 for autoencoder, 256 for diffusion), learning rates (5e-5 max for autoencoder, 1e-4 for diffusion), epochs (1,600 and 8,000), learning rate schedule (linear warmup + cosine decay), KL weight (0.001), M=512, C=512, C0=32. EDM default settings are also referenced."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a neural network training and evaluation paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 documents preprocessing: shapes converted to watertight meshes, normalized to bounding box, 500K surface points sampled, 500K occupancy points from bounding volume, 500K near-surface points. Shape completion preprocessing and image rendering setup are also described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.8 is explicitly titled 'Limitations' and provides substantive discussion of the two-stage training drawbacks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses only training cost and retraining needs -- these are practical limitations, not threats to the validity of the performance claims. No discussion of whether ShapeNet results generalize, metric limitations, single-run variance, or other validity threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. No statements bound claims to ShapeNet-v2 or note that results may not transfer to real-world scanned data, non-rigid objects, or other domains."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "ShapeNet-v2 is a publicly available dataset that enables independent verification. The specific train/val splits from Zhang et al. 2022 are referenced."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 describes data source (ShapeNet-v2), splits (from Zhang et al. 2022), and preprocessing pipeline. Additional data sources for specific tasks are described (3D-R2N2 renderings for image conditioning, ShapeGlot text prompts)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data is a standard public 3D shape benchmark (ShapeNet-v2)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 documents the full pipeline: raw ShapeNet meshes -> watertight conversion -> bounding box normalization -> 500K surface point sampling -> 500K bounding volume occupancy sampling + 500K near-surface sampling. Each step is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work was supported by the SDAIA-KAUST Center of Excellence in Data Science and Artificial Intelligence (SDAIA-KAUST AI) as well as the ERC Starting Grant Scan2CAD (804724).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Biao Zhang and Peter Wonka at KAUST; Jiapeng Tang and Matthias Niessner at TU Munich."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "SDAIA-KAUST AI (government-academic center) and the ERC (European Research Council) are academic/government funders with no financial stake in whether this 3D shape representation outperforms competitors."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests statement in the paper. Per the schema, 'If there is no competing interests statement at all, NO -- absence of disclosure is not the same as absence of conflict.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper trains its own neural networks from scratch on ShapeNet-v2. It does not evaluate a pre-trained foundation model's capability on a benchmark. Contamination questions are not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above -- the paper trains its own models with explicit train/val splits. Pre-training data contamination does not apply."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above -- no pre-trained model capabilities are being evaluated."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 18 denoising steps for sampling but does not report inference latency, wall-clock time per sample, or cost per generated shape."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 specifies hardware and training duration: 'The shape auto-encoder is trained on 8 A100, with batch size of 512 for T=1,600 epochs. The diffusion models are trained on 4 A100 with batch size of 256 for T=8,000 epochs.' The schema asks to look for 'GPU hours, total API spend, hardware used, training time.' Hardware used and training time are provided."
       }
     }
   }
 }