ai-research-survey

scan.json (17557B)

---

 {
   "paper": {
     "title": "ConCo: Optimizing Compilation of Concurrent Tensor Programs on Shared GPU",
     "authors": ["Jiamin Lu", "Jingwei Sun", "Yunlong Xu", "Peng Sun", "Guangzhong Sun"],
     "year": 2025,
     "venue": "ICS '25 (International Conference on Supercomputing)",
     "doi": "10.1145/3721145.3730433"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses operators from the ROLLER benchmark collection (publicly available), but does not release their own experimental data, configurations, or scripts."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions GPU types (RTX 2080 Ti, A100 80 GB) and CPUs but does not provide software versions, library versions, requirements files, or environment setup details beyond stating TVM/Ansor is used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 'up to 69.85%', '1.2x') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ConCo outperforms baselines but provides no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., JCT reduction of up to 69.85% compared to Baseline 1, throughput improvement of up to 1.2x, compilation time reduced by more than 2x. Tables 2-4 provide absolute latency values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why these particular models, operators, or number of experimental configurations were chosen. The number of operator pairs (500 repetitions) is stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. Results appear to be single-run or unreported aggregation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are defined: Baseline1 (ConCo 100% code under default MPS) and Baseline2 (ConCo 100% code with optimal resource allocation search). Ansor is also compared for compilation time."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Ansor (2020) and VELTAIR (2022) are discussed as the most relevant prior work. The paper argues VELTAIR cannot be applied to GPUs, and Ansor is the state-of-the-art DNN compiler used as baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 (Features Analysis) analyzes what makes ConCo-generated code different. The evaluation separates single-operator (Section 4.4) from end-to-end (Section 4.5) scenarios, and the compilation speedup is evaluated separately (Section 4.2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: JCT reduction, normalized throughput, inference latency, and compilation time."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant for a systems/compiler optimization paper measuring throughput and latency."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems paper, not a machine learning model evaluation. There is no train/test split concept applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (ResNet-18, VGG-19, BERT-small, DLRM), per GPU (2080 Ti, A100), per operator pair (Table 2), and per resource constraint level (Tables 3-4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 discusses why end-to-end gains are smaller than single-operator gains, attributing it to the inability to control GPU operator scheduling. This is an honest discussion of where the approach is limited."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that end-to-end performance improvements are inferior to single-operator improvements (Section 4.5) and explains why. It also shows VELTAIR's approach fails on GPUs (Section 2.4, Figure 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 1.2x throughput improvement, 69.85% JCT reduction, and 2x compilation speedup are all supported by results in Sections 4.2-4.5 and corresponding figures/tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims (ConCo improves throughput) supported by controlled experiments comparing code variants on the same hardware under the same workloads. Section 4.6 provides a feature analysis explaining why ConCo's code performs better."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to the tested GPUs (2080 Ti, A100), tested models (ResNet-18, VGG-19, BERT-small, DLRM), and MPS-based sharing. The conclusion explicitly states future work will extend to more hardware architectures."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements, such as whether the gains are specific to MPS or would hold under other sharing mechanisms, or whether Ansor's cost model could be retrained to achieve similar results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use LLMs or pre-trained models via API. The DNN models evaluated (ResNet-18, VGG-19, BERT-small, DLRM) are standard architectures compiled with TVM."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Compilation hyperparameters are reported: measuring_batch_size=64, N=len(tasks), M=5, alpha=2.5% (Section 4.2). Ansor configuration follows TVM docs (900 x len(tasks) trials). Operator configurations in Table 1."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how operators were selected from the ROLLER benchmark (Section 2.3), how batch sizes were adjusted (Table 1 note), and the experimental setup including concurrency levels (Table 5)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion briefly mentions future work but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Scope boundaries are only implicit (tested on 2 GPUs, 4 models)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (latency measurements, compilation logs) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The measurement procedure is described: operators run 500 times each (Section 4.4), compilation follows Ansor's process with specified hyperparameters, and the scheduling model is described (Section 4.5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard benchmarks and compiler-generated code."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The compilation pipeline (Figure 6) and evaluation pipeline are documented: models are partitioned into subgraphs, code variants generated across resource levels, then evaluated in concurrent scenarios."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Science and Technology of China and Independent Researchers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It is a systems paper about compiler optimization."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no pre-trained model benchmark evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no pre-trained model benchmark evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference latency is extensively reported across resource configurations (Tables 3-4), and compilation time is quantified in hours (Figure 8)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Compilation times are reported in detail: e.g., ResNet-18 takes 129.3 hours with ConCo vs 301.3 hours with Ansor on 2080 Ti (Figure 8). Hardware is specified (2080 Ti, A100 80 GB)."
       }
     }
   },
   "claims": [
     {
       "claim": "ConCo improves inference throughput by up to 1.2x compared to existing solutions in concurrent DNN inference on shared GPUs.",
       "evidence": "Figures 9 and 10 show normalized throughput across 24 task configurations on 2080 Ti (up to 1.21x) and A100 (up to 1.18x) compared to Baseline2.",
       "supported": "strong"
     },
     {
       "claim": "ConCo reduces job completion time by up to 69.85% for concurrent single-operator processes compared to Baseline 1.",
       "evidence": "Table 2 shows JCT reductions for operator pairs, with the maximum 69.85% reduction on A100 for ResNet-18 22nd OP vs BERT-small 6th OP.",
       "supported": "strong"
     },
     {
       "claim": "ConCo's optimal-code-sharing strategy achieves more than 2x compilation speedup over standard Ansor.",
       "evidence": "Figure 8 shows compilation times: e.g., ResNet-18 on 2080 Ti takes 129.3h (ConCo) vs 301.3h (Ansor). Section 4.2 states ConCo requires only 14.3%-32.3% of Ansor's time to achieve equivalent quality.",
       "supported": "strong"
     },
     {
       "claim": "Code that performs optimally in exclusive GPU scenarios is suboptimal in concurrent scenarios.",
       "evidence": "Figure 2 demonstrates that suboptimal exclusive codes can reduce concurrent completion time by up to 26.2% across 5 operator pairs (Section 2.3).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ConCo is a concurrency-aware DNN compilation framework that generates multiple code variants optimized for different GPU resource constraints and selects optimal variants at runtime. The key insight is that code optimized for exclusive GPU use is suboptimal under concurrent execution due to resource contention. ConCo achieves up to 1.2x throughput improvement and 69.85% JCT reduction for concurrent inference, while reducing compilation time by over 2x through an optimal-code-sharing strategy that exploits performance correlations across resource configurations.",
   "red_flags": [
     {
       "flag": "No variance or error bars reported",
       "detail": "All results are point estimates with no indication of measurement variance across runs. For latency measurements on GPUs, variance can be significant due to thermal throttling, scheduling jitter, and other factors."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations or threats to validity. Key limitations (MPS-only, limited model diversity, no real production workload evaluation) are not acknowledged."
     },
     {
       "flag": "'Up to' framing",
       "detail": "Key results are reported as 'up to' maximums (69.85% JCT reduction, 1.2x throughput). The median or average improvements are not clearly stated, which could overstate typical gains."
     }
   ],
   "cited_papers": [
     {
       "title": "TVM: An automated End-to-End optimizing compiler for deep learning",
       "authors": ["Tianqi Chen", "Thierry Moreau", "Ziheng Jiang"],
       "year": 2018,
       "relevance": "Foundational DNN compiler framework that ConCo extends for concurrent execution scenarios."
     },
     {
       "title": "Ansor: Generating High-Performance tensor programs for deep learning",
       "authors": ["Lianmin Zheng", "Chengfan Jia", "Minmin Sun"],
       "year": 2020,
       "relevance": "The auto-scheduling system that ConCo builds upon and compares against as primary baseline."
     },
     {
       "title": "VELTAIR: towards high-performance multi-tenant deep learning services via adaptive compilation and scheduling",
       "authors": ["Zihan Liu", "Jingwen Leng", "Zhihui Zhang"],
       "year": 2022,
       "relevance": "Prior work on compilation optimization for concurrent DNN inference on CPUs; ConCo addresses why this approach fails on GPUs."
     },
     {
       "title": "ROLLER: Fast and efficient tensor compilation for deep learning",
       "authors": ["Hongyu Zhu", "Ruofan Wu", "Yijia Diao"],
       "year": 2022,
       "relevance": "DNN compiler for operator-level optimization; provides the benchmark operator collection used in ConCo's evaluation."
     },
     {
       "title": "A survey of multi-tenant deep learning inference on gpu",
       "authors": ["Fuxun Yu", "Di Wang", "Longfei Shangguan"],
       "year": 2022,
       "relevance": "Survey of GPU sharing for DNN inference, providing context for the multi-tenant inference problem ConCo addresses."
     }
   ]
 }