ai-research-survey

scan.json (23557B)

---

 {
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "paper": {
     "title": "3DShape2VecSet: A 3D Shape Representation for Neural Fields and Generative Diffusion Models",
     "authors": [
       "Biao Zhang",
       "Jiapeng Tang",
       "Matthias Niessner",
       "Peter Wonka"
     ],
     "year": 2023,
     "venue": "ACM Transactions on Graphics (SIGGRAPH)",
     "arxiv_id": "2301.11445"
   },
   "methodology_tags": [
     "benchmark-eval"
   ],
   "claims": [
     {
       "claim": "3DShape2VecSet achieves state-of-the-art shape autoencoding on ShapeNet, outperforming OccNet, ConvOccNet, IF-Net, and 3DILG across IoU, Chamfer distance, and F-Score.",
       "evidence": "Table 3 shows mean IoU 0.965 (Point Queries) vs 0.953 (3DILG), mean Chamfer 0.038 vs 0.040, mean F-Score 0.970 vs 0.966 across all 55 categories.",
       "supported": "strong"
     },
     {
       "claim": "The latent set diffusion model achieves state-of-the-art unconditional 3D shape generation on full ShapeNet.",
       "evidence": "Table 6 shows Surface-FPD 0.76 (C0=32) vs 1.89 (3DILG), Rendering-FID 17.08 vs 24.83 (3DILG). Table 7 shows large margin over PVD (FPD 0.63 vs 2.33).",
       "supported": "strong"
     },
     {
       "claim": "Point Queries outperform Learned Queries for shape encoding.",
       "evidence": "Table 3 shows Point Queries consistently better across all 7 categories and all 3 metrics.",
       "supported": "strong"
     },
     {
       "claim": "First demonstration of text-conditioned 3D shape generation using diffusion models.",
       "evidence": "Section 8.4 with qualitative results in Fig. 11 only. No quantitative metrics provided.",
       "supported": "weak"
     },
     {
       "claim": "Category-conditioned generation shows improved recall over competing methods while maintaining comparable precision.",
       "evidence": "Table 9: recall 0.86 for chair vs 0.65 (3DILG) and 0.57 (NeuralWavelet); recall 0.89 for table vs 0.59 (3DILG).",
       "supported": "moderate"
     }
   ],
   "key_findings": "3DShape2VecSet proposes a novel 3D shape representation encoding neural fields as a fixed-size set of latent vectors without explicit spatial coordinates, processed via cross-attention. On ShapeNet-v2, it achieves state-of-the-art shape autoencoding (IoU 0.965 vs prior 0.953) and unconditional generation (FPD 0.76 vs 1.89). The paper demonstrates multiple conditional generation applications including text-, category-, image-, and point-cloud-conditioned shape generation using latent diffusion.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results across Tables 3-9 are single point estimates with no error bars, standard deviations, or multi-seed results. Impossible to assess whether improvements are within noise."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Comparative claims ('our results are best', 'beat PVD by a large margin') made without any statistical tests."
     },
     {
       "flag": "Text-conditioned generation claim unsupported quantitatively",
       "detail": "Section 8.4 claims 'first text-conditioned 3D generation using diffusion models' with only qualitative figure comparisons, no quantitative metrics."
     },
     {
       "flag": "Generalization bounded only to ShapeNet",
       "detail": "All experiments on ShapeNet-v2 (synthetic man-made objects) but claims framed broadly without bounding to this dataset."
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code URL provided in abstract: 'Code: https://1zb.github.io/3DShape2VecSet/'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available ShapeNet-v2, 3D-R2N2 renderings, and ShapeGlot text prompts. All standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware mentioned (8 A100 for autoencoder, 4 A100 for diffusion) but no requirements.txt, Dockerfile, or library version specifications provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. Implementation details in Section 7.3 but no runnable commands or README-style instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (3-9) report only point estimates. No confidence intervals, error bars, or ± notation found."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims of improvement are based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables provide absolute metric values for both proposed method and baselines (e.g., IoU 0.965 vs 0.953, FPD 0.76 vs 1.89), allowing readers to assess magnitude of improvement in context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset size or number of generated samples used for evaluation metrics."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviations, variance across seeds, or multiple-run results reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: OccNet, ConvOccNet, IF-Net, 3DILG for autoencoding; PVD, 3DILG, NeuralWavelet, Grid-83, 3DShapeGen, AutoSDF for generation (Section 7.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include 3DILG (2022), NeuralWavelet (2022), PVD (2021) — all contemporary for a 2023 paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 ablates M (512, 256, 128, 64); Table 5 ablates C0 (1-64); Table 3 compares Learned vs Point Queries."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Autoencoding: IoU, Chamfer, F-Score. Generation: FPD, KPD, FID, KID, Precision, Recall, MMD-CD, MMD-EMD, COV-CD, COV-EMD."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated shape quality. For generative modeling, human perceptual evaluation is relevant but was not included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states 'We use the training/val splits in [Zhang et al. 2022].' Section 8.1 references 'test split.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows per-category results for 7 largest ShapeNet categories plus overall mean. Tables 8-9 show per-category generation results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states 'We initially explored many variations... Ultimately, we could not improve on existing irregular grids.' Section 8.8 discusses limitations of two-stage training."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "C0=64 gives worse generation results than C0=32 (Table 6). Section 4 reports that tri-planes, frequency compositions, and factored representations failed to improve over irregular grids."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'improved performance in 3D shape encoding and generative modeling' supported by Tables 3-9. All claimed applications demonstrated."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims supported by controlled ablation studies: Tables 4, 5 show single-variable manipulation of M and C0. Learnable vs Point Queries comparison in Table 3."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Results only on ShapeNet-v2 (synthetic man-made objects) but claims framed broadly as '3D shape encoding and generative modeling' without bounding to this dataset."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. Does not consider whether improvements stem from increased model capacity, training duration, or other confounds vs the representation design."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific metrics (IoU, Chamfer distance, F-Score, FID, KID, FPD, KPD) and frames claims at the same granularity: 'improved performance in 3D shape encoding and generative modeling' as measured by these metrics. No broader framing (e.g., 'visual quality') is claimed beyond what is measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Trains own neural networks from scratch; does not use pre-trained LLM APIs with versioning concerns."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Does not use prompting. All models trained end-to-end."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3: batch sizes (512, 256), learning rates (5e-5, 1e-4), epochs (1600, 8000), warmup + cosine decay, KL weight 0.001, M=512, C=512, C0=32, 18 denoising steps. EDM defaults referenced."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used. Standard two-stage neural network training pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 7: shapes → watertight meshes → normalized to bounding box → 500K surface points, 500K occupancy points from volume, 500K near-surface. Rendering and text prompt sources documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.8 'Limitations' discusses drawbacks of two-stage training strategy."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Limitations section discusses training cost but not threats to validity of performance claims (e.g., ShapeNet-only evaluation, single-run variance, metric limitations)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statement of what results do NOT show. No bounding of claims to ShapeNet or noting potential non-transferability to real-world data."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "ShapeNet-v2 is publicly available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 describes ShapeNet-v2 with splits from Zhang et al. 2022 and full preprocessing pipeline."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is a standard public benchmark (ShapeNet-v2)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: ShapeNet meshes → watertight conversion → normalization → point cloud/occupancy sampling."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments: 'supported by the SDAIA-KAUST Center of Excellence in Data Science and AI as well as the ERC Starting Grant Scan2CAD (804724).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: KAUST and TU Munich. No commercial product evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "SDAIA-KAUST AI and ERC are academic/government funders with no financial stake in the results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Trains own models from scratch on ShapeNet. No pre-trained model capability evaluation; contamination not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same — trains own models on standard splits; pre-training contamination concept not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model capabilities evaluated. Benchmark contamination not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only 18 denoising steps mentioned. No wall-clock inference time, latency per shape, or cost per generation reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3: autoencoder trained on 8 A100 for 1600 epochs; diffusion on 4 A100 for 8000 epochs. Hardware and training duration provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed experiments. All results appear single-run. Generative metrics (FID, KID) are known to be seed-sensitive."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Number of experimental runs never stated. Results presented without indicating single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Ablation studies explore M and C0 values but no systematic search budget reported. Exploration appears selective."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4-6 show ablation results for M and C0; best configuration (M=512, C0=32) selected based on reported metrics with transparent criteria."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests performed, so multiple comparison correction not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors re-implement Grid-83 baseline and re-train PVD. No acknowledgment of potential bias from re-implementing competitors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison at matched compute budgets. Proposed method uses 8+4 A100 GPUs but baseline compute requirements not reported for comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "ShapeNet used as sole benchmark without discussing whether it adequately measures 3D shape generation quality. No construct validity discussion."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding or tool framework is involved. The paper trains and evaluates its own neural networks from scratch using standard training pipelines. Scaffolding confounds are not applicable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Models trained from scratch on ShapeNet with standard splits. No pre-trained model that could have seen test data."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Standard train/test evaluation; no pre-trained model being probed for knowledge of test data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ShapeNet train and test splits contain highly similar objects. Uses splits from Zhang et al. 2022 without analyzing potential non-independence."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable for train-from-scratch setup on standard benchmark with defined splits."
       }
     }
   },
   "cited_papers": [
     {
       "title": "Denoising Diffusion Probabilistic Models",
       "authors": [
         "Jonathan Ho",
         "Ajay Jain",
         "Pieter Abbeel"
       ],
       "year": 2020,
       "relevance": "Foundational diffusion model paper underpinning the 3D generation methodology."
     },
     {
       "title": "High-resolution image synthesis with latent diffusion models",
       "authors": [
         "Robin Rombach",
         "Andreas Blattmann",
         "Dominik Lorenz",
         "Patrick Esser",
         "Björn Ommer"
       ],
       "year": 2022,
       "relevance": "Latent diffusion approach directly adapted for 3D shape generation in this work."
     },
     {
       "title": "Elucidating the Design Space of Diffusion-Based Generative Models",
       "authors": [
         "Tero Karras",
         "Miika Aittala",
         "Timo Aila",
         "Samuli Laine"
       ],
       "year": 2022,
       "relevance": "EDM framework used for training details and denoising objective formulation."
     },
     {
       "title": "Attention is all you need",
       "authors": [
         "Ashish Vaswani"
       ],
       "year": 2017,
       "relevance": "Transformer attention mechanism core to the proposed latent set representation."
     },
     {
       "title": "3DILG: Irregular Latent Grids for 3D Generative Modeling",
       "authors": [
         "Biao Zhang",
         "Matthias Nießner",
         "Peter Wonka"
       ],
       "year": 2022,
       "relevance": "Directly preceding work from same authors; key baseline comparison."
     },
     {
       "title": "Neural wavelet-domain diffusion for 3d shape generation",
       "authors": [
         "Ka-Hei Hui",
         "Ruihui Li",
         "Jingyu Hu",
         "Chi-Wing Fu"
       ],
       "year": 2022,
       "relevance": "Key competing method (NeuralWavelet) used as generation baseline."
     },
     {
       "title": "Occupancy networks: Learning 3d reconstruction in function space",
       "authors": [
         "Lars Mescheder"
       ],
       "year": 2019,
       "relevance": "Foundational neural field method; autoencoding baseline."
     },
     {
       "title": "Perceiver: General perception with iterative attention",
       "authors": [
         "Andrew Jaegle"
       ],
       "year": 2021,
       "relevance": "Cross-attention architecture for compressing large inputs into fixed-size latent sets; directly inspired encoding design."
     },
     {
       "title": "BERT: Pre-training of deep bidirectional transformers for language understanding",
       "authors": [
         "Jacob Devlin"
       ],
       "year": 2018,
       "relevance": "Used as text encoder for text-conditioned 3D shape generation experiments."
     },
     {
       "title": "DiffusionSDF: Conditional Generative Modeling of Signed Distance Functions",
       "authors": [
         "Gene Chou",
         "Yuval Bahat",
         "Felix Heide"
       ],
       "year": 2022,
       "arxiv_id": "2211.13757",
       "relevance": "Concurrent work using diffusion in latent space for neural field generation."
     },
     {
       "title": "LION: Latent Point Diffusion Models for 3D Shape Generation",
       "authors": [
         "Xiaohui Zeng"
       ],
       "year": 2022,
       "arxiv_id": "2210.06978",
       "relevance": "Related diffusion model for 3D point cloud generation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Useful for 3D graphics researchers but requires significant expertise and compute to adapt to production workflows."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected trend that learned latent representations outperform hand-designed ones for 3D generation."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications in 3D shape representation research."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward incremental improvement over prior methods with no controversy or challenge to industry claims."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released but requires multi-GPU training on ShapeNet data, making casual reproduction impractical."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "KAUST and TU Munich are recognized in computer vision but are not household names in broader tech audiences."
     }
   }
 }