Add initContainer to download model

Add initContainer to download model to avoid the permission conflict issue of the .locks directory in the host modeUseHostPath path, when sharing the host model directory between different services. Signed-off-by: Lianhao Lu <[email protected]>
opea-project · Nov 27, 2024 · 8661d19 · 8661d19
1 parent 0f21681
commit 8661d19
Show file tree

Hide file tree

Showing 15 changed files with 162 additions and 9 deletions.
diff --git a/helm-charts/common/guardrails-usvc/README.md b/helm-charts/common/guardrails-usvc/README.md
@@ -47,11 +47,10 @@ curl http://localhost:9090/v1/guardrails \
 
 ## Values
 
-| Key                             | Type   | Default                              | Description                                                                                                                                                  |
-| ------------------------------- | ------ | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| global.HUGGINGFACEHUB_API_TOKEN | string | `""`                                 | Your own Hugging Face API token                                                                                                                              |
-| global.modelUseHostPath         | string | `"/mnt/opea-models"`                 | Cached models directory, tgi will not download if the model is cached here. The host path "modelUseHostPath" will be mounted to container as /data directory |
-| image.repository                | string | `"opea/guardrails-usvc"`             |                                                                                                                                                              |
-| service.port                    | string | `"9090"`                             |                                                                                                                                                              |
-| SAFETY_GUARD_ENDPOINT           | string | `""`                                 | LLM endpoint                                                                                                                                                 |
-| SAFETY_GUARD_MODEL_ID           | string | `"meta-llama/Meta-Llama-Guard-2-8B"` | Model ID for the underlying LLM service is using                                                                                                             |
+| Key                             | Type   | Default                              | Description                                      |
+| ------------------------------- | ------ | ------------------------------------ | ------------------------------------------------ |
+| global.HUGGINGFACEHUB_API_TOKEN | string | `""`                                 | Your own Hugging Face API token                  |
+| image.repository                | string | `"opea/guardrails-usvc"`             |                                                  |
+| service.port                    | string | `"9090"`                             |                                                  |
+| SAFETY_GUARD_ENDPOINT           | string | `""`                                 | LLM endpoint                                     |
+| SAFETY_GUARD_MODEL_ID           | string | `"meta-llama/Meta-Llama-Guard-2-8B"` | Model ID for the underlying LLM service is using |
diff --git a/helm-charts/common/speecht5/templates/configmap.yaml b/helm-charts/common/speecht5/templates/configmap.yaml
@@ -18,3 +18,4 @@ data:
   HF_ENDPOINT: {{ .Values.global.HF_ENDPOINT | quote}}
   {{- end }}
   HUGGINGFACE_HUB_CACHE: "/data"
+  HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}}
diff --git a/helm-charts/common/speecht5/templates/deployment.yaml b/helm-charts/common/speecht5/templates/deployment.yaml
@@ -27,6 +27,32 @@ spec:
       {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- if not (hasPrefix "/data/" .Values.TTS_MODEL_PATH) }}
+      initContainers:
+        - name: model-downloader
+          envFrom:
+            - configMapRef:
+                name: {{ include "speecht5.fullname" . }}-config
+          securityContext:
+            {{- if .Values.global.modelUseHostPath }}
+            {}
+            {{- else }}
+            {{- toYaml .Values.securityContext | nindent 12 }}
+            {{- end }}
+          image: huggingface/downloader:0.17.3
+          command: ['sh', '-c']
+          args:
+            - |
+              huggingface-cli download --cache-dir /data --token $(HF_TOKEN)  {{ .Values.TTS_MODEL_PATH | quote }};
+              huggingface-cli download --cache-dir /data --token $(HF_TOKEN)  {{ .Values.VOCODER_MODEL| quote }};
+              {{- if .Values.global.modelUseHostPath }}
+              chown -R 1000 /data/models--{{ replace "/" "--" .Values.TTS_MODEL_PATH }};
+              chown -R 1000 /data/models--{{ replace "/" "--" .Values.VOCODER_MODEL }}
+              {{- end }}
+          volumeMounts:
+            - mountPath: /data
+              name: model-volume
+      {{- end }}
       containers:
         - name: {{ .Release.Name }}
           envFrom:

diff --git a/helm-charts/common/speecht5/values.yaml b/helm-charts/common/speecht5/values.yaml
@@ -8,7 +8,7 @@
 replicaCount: 1
 
 TTS_MODEL_PATH: "microsoft/speecht5_tts"
-# VOCODE_MODEL: "microsoft/speecht5_hifigan"
+VOCODER_MODEL: "microsoft/speecht5_hifigan"
 
 image:
   repository: opea/speecht5
@@ -85,6 +85,7 @@ global:
   http_proxy: ""
   https_proxy: ""
   no_proxy: ""
+  HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here"
 
   # Choose where to save your downloaded models
   # Set modelUseHostPath for local directory, this is good for one node test. Example:

diff --git a/helm-charts/common/tei/templates/configmap.yaml b/helm-charts/common/tei/templates/configmap.yaml
@@ -32,3 +32,4 @@ data:
   {{- if .Values.MAX_WARMUP_SEQUENCE_LENGTH }}
   MAX_WARMUP_SEQUENCE_LENGTH: {{ .Values.MAX_WARMUP_SEQUENCE_LENGTH | quote }}
   {{- end }}
+  HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}}
diff --git a/helm-charts/common/tei/templates/deployment.yaml b/helm-charts/common/tei/templates/deployment.yaml
@@ -30,6 +30,30 @@ spec:
       {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- if not (hasPrefix "/data/" .Values.EMBEDDING_MODEL_ID) }}
+      initContainers:
+        - name: model-downloader
+          envFrom:
+            - configMapRef:
+                name: {{ include "tei.fullname" . }}-config
+          securityContext:
+            {{- if .Values.global.modelUseHostPath }}
+            {}
+            {{- else }}
+            {{- toYaml .Values.securityContext | nindent 12 }}
+            {{- end }}
+          image: huggingface/downloader:0.17.3
+          command: ['huggingface-cli', 'download']
+          args:
+            - "--cache-dir"
+            - "/data"
+            - "--token"
+            - $(HF_TOKEN)
+            - $(MODEL_ID)
+          volumeMounts:
+            - mountPath: /data
+              name: model-volume
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           envFrom:

diff --git a/helm-charts/common/tei/values.yaml b/helm-charts/common/tei/values.yaml
@@ -99,6 +99,7 @@ global:
   http_proxy: ""
   https_proxy: ""
   no_proxy: ""
+  HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here"
 
   # Choose where to save your downloaded models
   # Set modelUseHostPath for local directory, this is good for one node test. Example:

diff --git a/helm-charts/common/teirerank/templates/configmap.yaml b/helm-charts/common/teirerank/templates/configmap.yaml
@@ -32,3 +32,4 @@ data:
   {{- if .Values.MAX_WARMUP_SEQUENCE_LENGTH }}
   MAX_WARMUP_SEQUENCE_LENGTH: {{ .Values.MAX_WARMUP_SEQUENCE_LENGTH | quote }}
   {{- end }}
+  HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}}
diff --git a/helm-charts/common/teirerank/templates/deployment.yaml b/helm-charts/common/teirerank/templates/deployment.yaml
@@ -30,6 +30,30 @@ spec:
       {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- if not (hasPrefix "/data/" .Values.RERANK_MODEL_ID) }}
+      initContainers:
+        - name: model-downloader
+          envFrom:
+            - configMapRef:
+                name: {{ include "teirerank.fullname" . }}-config
+          securityContext:
+            {{- if .Values.global.modelUseHostPath }}
+            {}
+            {{- else }}
+            {{- toYaml .Values.securityContext | nindent 12 }}
+            {{- end }}
+          image: huggingface/downloader:0.17.3
+          command: ['huggingface-cli', 'download']
+          args:
+            - "--cache-dir"
+            - "/data"
+            - "--token"
+            - $(HF_TOKEN)
+            - $(MODEL_ID)
+          volumeMounts:
+            - mountPath: /data
+              name: model-volume
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           envFrom:

diff --git a/helm-charts/common/teirerank/values.yaml b/helm-charts/common/teirerank/values.yaml
@@ -99,6 +99,7 @@ global:
   http_proxy: ""
   https_proxy: ""
   no_proxy: ""
+  HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here"
 
   # Choose where to save your downloaded models
   # Set modelUseHostPath for local directory, this is good for one node test. Example:

diff --git a/helm-charts/common/tgi/templates/deployment.yaml b/helm-charts/common/tgi/templates/deployment.yaml
@@ -30,6 +30,30 @@ spec:
       {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- if not (hasPrefix "/data/" .Values.LLM_MODEL_ID) }}
+      initContainers:
+        - name: model-downloader
+          envFrom:
+            - configMapRef:
+                name: {{ include "tgi.fullname" . }}-config
+          securityContext:
+            {{- if .Values.global.modelUseHostPath }}
+            {}
+            {{- else }}
+            {{- toYaml .Values.securityContext | nindent 12 }}
+            {{- end }}
+          image: huggingface/downloader:0.17.3
+          command: ['huggingface-cli', 'download']
+          args:
+            - "--cache-dir"
+            - "/data"
+            - "--token"
+            - $(HF_TOKEN)
+            - $(MODEL_ID)
+          volumeMounts:
+            - mountPath: /data
+              name: model-volume
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           envFrom:

diff --git a/helm-charts/common/vllm/templates/deployment.yaml b/helm-charts/common/vllm/templates/deployment.yaml
@@ -27,6 +27,30 @@ spec:
       {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- if not (hasPrefix "/data/" .Values.LLM_MODEL_ID) }}
+      initContainers:
+        - name: model-downloader
+          envFrom:
+            - configMapRef:
+                name: {{ include "vllm.fullname" . }}-config
+          securityContext:
+            {{- if .Values.global.modelUseHostPath }}
+            {}
+            {{- else }}
+            {{- toYaml .Values.securityContext | nindent 12 }}
+            {{- end }}
+          image: huggingface/downloader:0.17.3
+          command: ['huggingface-cli', 'download']
+          args:
+            - "--cache-dir"
+            - "/data"
+            - "--token"
+            - $(HF_TOKEN)
+            - {{ .Values.LLM_MODEL_ID | quote }}
+          volumeMounts:
+            - mountPath: /data
+              name: model-volume
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           envFrom:

diff --git a/helm-charts/common/whisper/templates/configmap.yaml b/helm-charts/common/whisper/templates/configmap.yaml
@@ -18,3 +18,4 @@ data:
   HF_ENDPOINT: {{ .Values.global.HF_ENDPOINT | quote}}
   {{- end }}
   HUGGINGFACE_HUB_CACHE: "/data"
+  HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}}
diff --git a/helm-charts/common/whisper/templates/deployment.yaml b/helm-charts/common/whisper/templates/deployment.yaml
@@ -27,6 +27,30 @@ spec:
       {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- if not (hasPrefix "/data/" .Values.ASR_MODEL_PATH) }}
+      initContainers:
+        - name: model-downloader
+          envFrom:
+            - configMapRef:
+                name: {{ include "whisper.fullname" . }}-config
+          securityContext:
+            {{- if .Values.global.modelUseHostPath }}
+            {}
+            {{- else }}
+            {{- toYaml .Values.securityContext | nindent 12 }}
+            {{- end }}
+          image: huggingface/downloader:0.17.3
+          command: ['sh', '-c']
+          args:
+            - |
+              huggingface-cli download --cache-dir /data --token $(HF_TOKEN)  {{ .Values.ASR_MODEL_PATH | quote }};
+              {{- if .Values.global.modelUseHostPath }}
+              chown -R 1000 /data/models--{{ replace "/" "--" .Values.ASR_MODEL_PATH }}
+              {{- end }}
+          volumeMounts:
+            - mountPath: /data
+              name: model-volume
+      {{- end }}
       containers:
         - name: {{ .Release.Name }}
           envFrom:

diff --git a/helm-charts/common/whisper/values.yaml b/helm-charts/common/whisper/values.yaml
@@ -84,6 +84,7 @@ global:
   http_proxy: ""
   https_proxy: ""
   no_proxy: ""
+  HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here"
 
   # Choose where to save your downloaded models
   # Set modelUseHostPath for local directory, this is good for one node test. Example: