Fix Nvidia implementation github action, support mlperf-inference-sub…

…mission-preprocessor-args
mlcommons · Feb 11, 2025 · 37f7ff2 · 37f7ff2
1 parent aa25ac3
commit 37f7ff2
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 4 deletions.
diff --git a/.github/workflows/test-nvidia-mlperf-inference-implementations.yml b/.github/workflows/test-nvidia-mlperf-inference-implementations.yml
@@ -45,6 +45,11 @@ jobs:
             gpu_name=rtx_4090
             docker_string=" --docker"
           fi
+          if [ "${{ matrix.model }}" = "bert-99.9" ]; then
+            submission_preprocessor_args=" --noinfer-low-accuracy-results"
+          else
+            submission_preprocessor_args=""
+          fi
           category="datacenter,edge"
           if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi
           python3 -m venv gh_action
@@ -53,6 +58,6 @@ jobs:
           pip install --upgrade mlcflow
           mlc pull repo mlcommons@mlperf-automations --branch=dev
           
-          mlcr --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r5.0-dev --preprocess_submission=yes --pull_changes=yes --pull_inference_changes=yes --execution_mode=valid --gpu_name=$gpu_name --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=$hw_name --implementation=nvidia --backend=tensorrt --category=$category --division=closed  --docker_dt --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --adr.compiler.tags=gcc --device=cuda --use_model_from_host=yes --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  $docker_string --quiet
+          mlcr --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r5.0-dev --preprocess_submission=yes --pull_changes=yes --pull_inference_changes=yes --execution_mode=valid --gpu_name=$gpu_name --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=$hw_name --implementation=nvidia --backend=tensorrt --category=$category --division=closed  --docker_dt --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --adr.compiler.tags=gcc --device=cuda --use_model_from_host=yes --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  $docker_string $submission_preprocessor_args --quiet
 
           mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from GH action on NVIDIA_$hw_name" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=$hw_name
diff --git a/script/generate-mlperf-inference-submission/meta.yaml b/script/generate-mlperf-inference-submission/meta.yaml
@@ -85,6 +85,7 @@ input_mapping:
   sw_notes_extra: MLC_MLPERF_SUT_SW_NOTES_EXTRA
   tar: MLC_TAR_SUBMISSION_DIR
   get_platform_details: MLC_GET_PLATFORM_DETAILS
+  submission_preprocessor_args: MLC_MLPERF_PREPROCESS_SUBMISSION_EXTRA_ARGS
   version: MLC_MLPERF_SUBMISSION_CHECKER_VERSION
 post_deps:
 - enable_if_env:

diff --git a/...mplementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml b/...mplementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml
@@ -39,4 +39,4 @@
     Offline:
       target_qps: 1.3
     Server:
-      target_qps: 0.4
+      target_qps: 0.25
diff --git a/script/preprocess-mlperf-inference-submission/customize.py b/script/preprocess-mlperf-inference-submission/customize.py
@@ -2,7 +2,7 @@
 import os
 from os.path import exists
 import shutil
-
+from utils import *
 
 def preprocess(i):
 
@@ -31,8 +31,20 @@ def preprocess(i):
     version = env.get('MLC_MLPERF_SUBMISSION_CHECKER_VERSION', '')
     x_version = ' --version ' + version + ' ' if version != '' else ''
 
+    extra_args = []
+    if is_true(env.get('MLC_MLPERF_NOINFER_LOW_ACCURACY_RESULTS')):
+        extra_args.append("--noinfer-low-accuracy-results")
+    if is_true(env.get('MLC_MLPERF_NODELETE_EMPTY_DIRS')):
+        extra_args.append("--nodelete-empty-dirs")
+    if is_true(env.get('MLC_MLPERF_NOMOVE_FAILED_TO_OPEN')):
+        extra_args.append("--nomove-failed-to-open")
+    if is_true(env.get('MLC_MLPERF_NODELETE_FAILED')):
+        extra_args.append("--nodelete-failed")
+    if env.get('MLC_MLPERF_PREPROCESS_SUBMISSION_EXTRA_ARGS', '') != '':
+        extra_args.append(env['MLC_MLPERF_PREPROCESS_SUBMISSION_EXTRA_ARGS'])
+
     CMD = env['MLC_PYTHON_BIN'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "tools", "submission",
-                                                      "preprocess_submission.py") + "' --input '" + submission_dir + "' --submitter '" + submitter + "' --output '" + submission_processed + "'" + x_version
+                                                      "preprocess_submission.py") + "' --input '" + submission_dir + "' --submitter '" + submitter + "' --output '" + submission_processed + "'" + x_version + " " + " ".join(extra_args)
     env['MLC_RUN_CMD'] = CMD
 
     return {'return': 0}

diff --git a/script/preprocess-mlperf-inference-submission/meta.yaml b/script/preprocess-mlperf-inference-submission/meta.yaml
@@ -24,6 +24,7 @@ input_mapping:
   submission_dir: MLC_MLPERF_INFERENCE_SUBMISSION_DIR
   version: MLC_MLPERF_SUBMISSION_CHECKER_VERSION
   submitter: MLC_MLPERF_SUBMITTER
+  submission_preprocessor_args: MLC_MLPERF_PREPROCESS_SUBMISSION_EXTRA_ARGS
 tags:
 - run
 - mlc