Results from self hosted Github actions - NVIDIARTX4090

open/MLCommons/measurements/94754266090d-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/README.md

@@ -0,0 +1,3 @@
+| Model               | Scenario   | Accuracy   |   Throughput | Latency (in ms)   |
+|---------------------|------------|------------|--------------|-------------------|
+| stable-diffusion-xl | offline    | ()         |        0.384 | -                 |

open/MLCommons/measurements/94754266090d-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/94754266090d-reference-gpu-pytorch-v2.5.1-scc24-base_cu124.json

@@ -0,0 +1,7 @@
+{
+  "starting_weights_filename": "https://github.com/mlcommons/inference/tree/master/text_to_image#download-model",
+  "retraining": "no",
+  "input_data_types": "fp32",
+  "weight_data_types": "fp32",
+  "weight_transformations": "no"
+}

open/MLCommons/measurements/94754266090d-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/README.md

@@ -0,0 +1,94 @@
+*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.*
+
+## Host platform
+
+* OS version: Linux-6.8.0-49-generic-x86_64-with-glibc2.35
+* CPU version: x86_64
+* Python version: 3.10.12 (main, Nov  6 2024, 20:22:13) [GCC 11.4.0]
+* MLCommons CM version: 3.5.2
+
+## CM Run Command
+
+See [CM installation guide](https://docs.mlcommons.org/inference/install/).
+
+```bash
+pip install -U cmind
+
+cm rm cache -f
+
+cm pull repo mlcommons@mlperf-automations --checkout=48ea6b46a7606d1c5d74909e94d5599dbe7ff9e1
+
+cm run script \
+	--tags=app,mlperf,inference,generic,_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline \
+	--quiet=true \
+	--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes \
+	--env.CM_QUIET=yes \
+	--env.CM_MLPERF_IMPLEMENTATION=reference \
+	--env.CM_MLPERF_MODEL=sdxl \
+	--env.CM_MLPERF_RUN_STYLE=test \
+	--env.CM_MLPERF_SKIP_SUBMISSION_GENERATION=False \
+	--env.CM_DOCKER_PRIVILEGED_MODE=True \
+	--env.CM_MLPERF_BACKEND=pytorch \
+	--env.CM_MLPERF_SUBMISSION_SYSTEM_TYPE=datacenter \
+	--env.CM_MLPERF_CLEAN_ALL=True \
+	--env.CM_MLPERF_DEVICE=cuda \
+	--env.CM_MLPERF_USE_DOCKER=True \
+	--env.CM_MLPERF_MODEL_PRECISION=float16 \
+	--env.OUTPUT_BASE_DIR=/cm-mount/home/arjun/scc_gh_action_results \
+	--env.CM_MLPERF_LOADGEN_SCENARIO=Offline \
+	--env.CM_MLPERF_INFERENCE_SUBMISSION_DIR=/cm-mount/home/arjun/scc_gh_action_submissions \
+	--env.CM_MLPERF_INFERENCE_VERSION=5.0-dev \
+	--env.CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS=r4.1-dev_default \
+	--env.CM_MLPERF_SUBMISSION_DIVISION=open \
+	--env.CM_RUN_MLPERF_SUBMISSION_PREPROCESSOR=False \
+	--env.CM_MLPERF_SUBMISSION_GENERATION_STYLE=short \
+	--env.CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX4=scc24-base \
+	--env.CM_DOCKER_IMAGE_NAME=scc24-reference \
+	--env.CM_MLPERF_INFERENCE_MIN_QUERY_COUNT=50 \
+	--env.CM_MLPERF_LOADGEN_ALL_MODES=yes \
+	--env.CM_MLPERF_INFERENCE_SOURCE_VERSION=5.0.4 \
+	--env.CM_MLPERF_LAST_RELEASE=v5.0 \
+	--env.CM_TMP_PIP_VERSION_STRING= \
+	--env.CM_MODEL=sdxl \
+	--env.CM_MLPERF_LOADGEN_COMPLIANCE=no \
+	--env.CM_MLPERF_CLEAN_SUBMISSION_DIR=yes \
+	--env.CM_RERUN=yes \
+	--env.CM_MLPERF_LOADGEN_EXTRA_OPTIONS= \
+	--env.CM_MLPERF_LOADGEN_MODE=performance \
+	--env.CM_MLPERF_LOADGEN_SCENARIOS,=Offline \
+	--env.CM_MLPERF_LOADGEN_MODES,=performance,accuracy \
+	--env.CM_OUTPUT_FOLDER_NAME=test_results \
+	--env.CM_DOCKER_REUSE_EXISTING_CONTAINER=no \
+	--env.CM_DOCKER_DETACHED_MODE=yes \
+	--add_deps_recursive.get-mlperf-inference-results-dir.tags=_version.r4_1-dev \
+	--add_deps_recursive.get-mlperf-inference-submission-dir.tags=_version.r4_1-dev \
+	--add_deps_recursive.mlperf-inference-nvidia-scratch-space.tags=_version.r4_1-dev \
+	--add_deps_recursive.submission-checker.tags=_short-run \
+	--add_deps_recursive.coco2014-preprocessed.tags=_size.50,_with-sample-ids \
+	--add_deps_recursive.coco2014-dataset.tags=_size.50,_with-sample-ids \
+	--add_deps_recursive.nvidia-preprocess-data.extra_cache_tags=scc24-base \
+	--v=False \
+	--print_env=False \
+	--print_deps=False \
+	--dump_version_info=True
+```
+*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf (CM scripts),
+ you should simply reload mlcommons@mlperf-automations without checkout and clean CM cache as follows:*
+
+```bash
+cm rm repo mlcommons@mlperf-automations
+cm pull repo mlcommons@mlperf-automations
+cm rm cache -f
+
+```
+
+## Results
+
+Platform: 94754266090d-reference-gpu-pytorch-v2.5.1-scc24-base_cu124
+
+Model Precision: fp32
+
+### Accuracy Results 
+
+### Performance Results 
+`Samples per second`: `0.384298`

open/MLCommons/measurements/94754266090d-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/cm-deps.mmd

@@ -0,0 +1,76 @@
+graph TD
+    app-mlperf-inference,d775cac873ee4231_(_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline_) --> detect,os
+    app-mlperf-inference,d775cac873ee4231_(_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline_) --> get,sys-utils-cm
+    app-mlperf-inference,d775cac873ee4231_(_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline_) --> get,python
+    app-mlperf-inference,d775cac873ee4231_(_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline_) --> get,mlcommons,inference,src
+    get-mlperf-inference-utils,e341e5f86d8342e5 --> get,mlperf,inference,src
+    app-mlperf-inference,d775cac873ee4231_(_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline_) --> get,mlperf,inference,utils
+    get-cuda,46d133d9ef92422d_(_toolkit_) --> detect,os
+    get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,cuda,_toolkit
+    get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,python3
+    get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
+    get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> detect,cpu
+    get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3
+    get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3
+    get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,generic-python-lib,_pip
+    get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,generic-python-lib,_package.pycuda
+    get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
+    get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> detect,cpu
+    get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> get,python3
+    get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3
+    get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> get,generic-python-lib,_pip
+    get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,generic-python-lib,_package.numpy
+    app-mlperf-inference,d775cac873ee4231_(_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline_) --> get,cuda-devices,_with-pycuda
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> detect,cpu
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,sys-utils-cm
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,python
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,cuda,_cudnn
+    get-generic-python-lib,94b62a682bc44791_(_torch_cuda_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_torch_cuda
+    get-generic-python-lib,94b62a682bc44791_(_torchvision_cuda_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_torchvision_cuda
+    download-and-extract,c67e81a4ce2649f5_(_rclone,_url.mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp16_) --> download,file,_rclone,_url.mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp16
+    get-ml-model-stable-diffusion,22c6516b2d4d4c23_(_pytorch,_fp16_) --> download-and-extract,_rclone,_url.mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp16
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,ml-model,stable-diffusion,text-to-image,sdxl,raw,_pytorch,_fp16
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,dataset,coco2014,_validation,_with-sample-ids,_size.50
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,cpu
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> get,python
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> get,mlcommons,inference,src
+    get-mlperf-inference-sut-configs,c2fbf72009e2445b --> get,cache,dir,_name.mlperf-inference-sut-configs
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> get,sut,configs
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> generate,user-conf,mlperf,inference
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,loadgen
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,mlcommons,inference,src
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,mlcommons,inference,src
+    get-generic-python-lib,94b62a682bc44791_(_package.psutil_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.psutil
+    get-generic-python-lib,94b62a682bc44791_(_package.diffusers_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.diffusers
+    get-generic-python-lib,94b62a682bc44791_(_package.transformers_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.transformers
+    get-generic-python-lib,94b62a682bc44791_(_package.torchvision_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.torchvision
+    get-generic-python-lib,94b62a682bc44791_(_package.accelerate_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.accelerate
+    get-generic-python-lib,94b62a682bc44791_(_package.torchmetrics_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.torchmetrics
+    get-generic-python-lib,94b62a682bc44791_(_package.torch-fidelity_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.torch-fidelity
+    get-generic-python-lib,94b62a682bc44791_(_package.open_clip_torch_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.open_clip_torch
+    get-generic-python-lib,94b62a682bc44791_(_package.opencv-python_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.opencv-python
+    get-generic-python-lib,94b62a682bc44791_(_package.scipy_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.scipy
+    get-generic-python-lib,94b62a682bc44791_(_package.pandas_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> get,generic-python-lib,_package.pandas
+    detect-cpu,586c8a43320142f7 --> detect,os
+    benchmark-program,19f369ef47084895 --> detect,cpu
+    benchmark-program-mlperf,cfff0132a8aa4018 --> benchmark-program,program
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_sdxl,_cuda,_offline,_pytorch,_float16_) --> benchmark-mlperf