Set num_workers to 1, #801 has landed (#370)

basetenlabs · Oct 29, 2024 · b595d5f · b595d5f
1 parent cd73c6a
commit b595d5f
Show file tree

Hide file tree

Showing 7 changed files with 13 additions and 15 deletions.
diff --git a/mistral/mistral-7b-instruct-chat-trt-llm-h100/config.yaml b/mistral/mistral-7b-instruct-chat-trt-llm-h100/config.yaml
@@ -35,7 +35,7 @@ resources:
   accelerator: H100
   use_gpu: true
 runtime:
-  num_workers: 2
+  num_workers: 1
   predict_concurrency: 256
 secrets: {}
 system_packages: []
diff --git a/mistral/mistral-7b-instruct-chat-trt-llm-smooth-quant/config.yaml b/mistral/mistral-7b-instruct-chat-trt-llm-smooth-quant/config.yaml
@@ -34,7 +34,7 @@ resources:
   accelerator: A100
   use_gpu: true
 runtime:
-  num_workers: 2
+  num_workers: 1
   predict_concurrency: 256
 secrets: {}
 system_packages: []
diff --git a/mistral/mistral-7b-instruct-chat-trt-llm-weights-only-quant-h100/config.yaml b/mistral/mistral-7b-instruct-chat-trt-llm-weights-only-quant-h100/config.yaml
@@ -35,7 +35,7 @@ resources:
   accelerator: H100
   use_gpu: true
 runtime:
-  num_workers: 2
+  num_workers: 1
   predict_concurrency: 256
 secrets: {}
 system_packages: []
diff --git a/mistral/mistral-7b-instruct-chat-trt-llm-weights-only-quant/config.yaml b/mistral/mistral-7b-instruct-chat-trt-llm-weights-only-quant/config.yaml
@@ -34,7 +34,7 @@ resources:
   accelerator: A100
   use_gpu: true
 runtime:
-  num_workers: 2
+  num_workers: 1
   predict_concurrency: 256
 secrets: {}
 system_packages: []
diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/config.yaml b/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/config.yaml
@@ -35,7 +35,7 @@ resources:
   accelerator: A100
   use_gpu: true
 runtime:
-  num_workers: 2
+  num_workers: 1
   predict_concurrency: 256
 secrets:
   hf_access_token: "ENTER HF ACCESS TOKEN HERE"

diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/config.yaml b/mistral/mixtral-8x7b-instruct-trt-llm/config.yaml
@@ -34,7 +34,7 @@ resources:
   accelerator: A100:2
   use_gpu: true
 runtime:
-  num_workers: 2
+  num_workers: 1
   predict_concurrency: 256
 secrets: {}
 system_packages: []
diff --git a/templates/generate.yaml b/templates/generate.yaml
@@ -25,7 +25,7 @@ mistral/mistral-7b-instruct-chat-trt-llm:
     - transformers==4.34.1
     runtime:
       predict_concurrency: 256
-      num_workers: 2
+      num_workers: 1
   ignore:
     - README.md
   template:
@@ -59,8 +59,7 @@ mistral/mistral-7b-instruct-chat-trt-llm-h100:
     - jinja2==3.1.3
     runtime:
       predict_concurrency: 256
-      # Drop num_workers once basetenlabs/truss#801 lands
-      num_workers: 2
+      num_workers: 1
     resources:
       accelerator: H100
   ignore:
@@ -96,8 +95,7 @@ mistral/mistral-7b-instruct-chat-trt-llm-weights-only-quant-h100:
     - jinja2==3.1.3
     runtime:
       predict_concurrency: 256
-      # Drop num_workers once basetenlabs/truss#801 lands
-      num_workers: 2
+      num_workers: 1
     resources:
       accelerator: H100
   ignore:
@@ -132,7 +130,7 @@ mistral/mistral-7b-instruct-chat-trt-llm-weights-only-quant:
     - transformers==4.34.1
     runtime:
       predict_concurrency: 256
-      num_workers: 2
+      num_workers: 1
   ignore:
     - README.md
   template:
@@ -165,7 +163,7 @@ mistral/mistral-7b-instruct-chat-trt-llm-smooth-quant:
     - transformers==4.34.1
     runtime:
       predict_concurrency: 256
-      num_workers: 2
+      num_workers: 1
   ignore:
     - README.md
   template:
@@ -198,7 +196,7 @@ mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant:
     - transformers==4.36.0
     runtime:
       predict_concurrency: 256
-      num_workers: 2
+      num_workers: 1
   ignore:
     - README.md
   template:
@@ -270,7 +268,7 @@ mistral/mixtral-8x7b-instruct-trt-llm:
     - transformers==4.36.0
     runtime:
       predict_concurrency: 256
-      num_workers: 2
+      num_workers: 1
     resources:
       accelerator: A100:2
   ignore: