From 8661d191bf327a6ee769fd42b94c0bd80a8bb2c0 Mon Sep 17 00:00:00 2001 From: Lianhao Lu Date: Wed, 27 Nov 2024 03:15:23 +0000 Subject: [PATCH] Add initContainer to download model Add initContainer to download model to avoid the permission conflict issue of the .locks directory in the host modeUseHostPath path, when sharing the host model directory between different services. Signed-off-by: Lianhao Lu --- helm-charts/common/guardrails-usvc/README.md | 15 +++++------ .../common/speecht5/templates/configmap.yaml | 1 + .../common/speecht5/templates/deployment.yaml | 26 +++++++++++++++++++ helm-charts/common/speecht5/values.yaml | 3 ++- .../common/tei/templates/configmap.yaml | 1 + .../common/tei/templates/deployment.yaml | 24 +++++++++++++++++ helm-charts/common/tei/values.yaml | 1 + .../common/teirerank/templates/configmap.yaml | 1 + .../teirerank/templates/deployment.yaml | 24 +++++++++++++++++ helm-charts/common/teirerank/values.yaml | 1 + .../common/tgi/templates/deployment.yaml | 24 +++++++++++++++++ .../common/vllm/templates/deployment.yaml | 24 +++++++++++++++++ .../common/whisper/templates/configmap.yaml | 1 + .../common/whisper/templates/deployment.yaml | 24 +++++++++++++++++ helm-charts/common/whisper/values.yaml | 1 + 15 files changed, 162 insertions(+), 9 deletions(-) diff --git a/helm-charts/common/guardrails-usvc/README.md b/helm-charts/common/guardrails-usvc/README.md index 0cf3b5656..969fb2a4b 100644 --- a/helm-charts/common/guardrails-usvc/README.md +++ b/helm-charts/common/guardrails-usvc/README.md @@ -47,11 +47,10 @@ curl http://localhost:9090/v1/guardrails \ ## Values -| Key | Type | Default | Description | -| ------------------------------- | ------ | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| global.HUGGINGFACEHUB_API_TOKEN | string | `""` | Your own Hugging Face API token | -| global.modelUseHostPath | string | `"/mnt/opea-models"` | Cached models directory, tgi will not download if the model is cached here. The host path "modelUseHostPath" will be mounted to container as /data directory | -| image.repository | string | `"opea/guardrails-usvc"` | | -| service.port | string | `"9090"` | | -| SAFETY_GUARD_ENDPOINT | string | `""` | LLM endpoint | -| SAFETY_GUARD_MODEL_ID | string | `"meta-llama/Meta-Llama-Guard-2-8B"` | Model ID for the underlying LLM service is using | +| Key | Type | Default | Description | +| ------------------------------- | ------ | ------------------------------------ | ------------------------------------------------ | +| global.HUGGINGFACEHUB_API_TOKEN | string | `""` | Your own Hugging Face API token | +| image.repository | string | `"opea/guardrails-usvc"` | | +| service.port | string | `"9090"` | | +| SAFETY_GUARD_ENDPOINT | string | `""` | LLM endpoint | +| SAFETY_GUARD_MODEL_ID | string | `"meta-llama/Meta-Llama-Guard-2-8B"` | Model ID for the underlying LLM service is using | diff --git a/helm-charts/common/speecht5/templates/configmap.yaml b/helm-charts/common/speecht5/templates/configmap.yaml index d488542fb..c153143f7 100644 --- a/helm-charts/common/speecht5/templates/configmap.yaml +++ b/helm-charts/common/speecht5/templates/configmap.yaml @@ -18,3 +18,4 @@ data: HF_ENDPOINT: {{ .Values.global.HF_ENDPOINT | quote}} {{- end }} HUGGINGFACE_HUB_CACHE: "/data" + HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}} diff --git a/helm-charts/common/speecht5/templates/deployment.yaml b/helm-charts/common/speecht5/templates/deployment.yaml index 6b5976b84..db5b9cd15 100644 --- a/helm-charts/common/speecht5/templates/deployment.yaml +++ b/helm-charts/common/speecht5/templates/deployment.yaml @@ -27,6 +27,32 @@ spec: {{- end }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if not (hasPrefix "/data/" .Values.TTS_MODEL_PATH) }} + initContainers: + - name: model-downloader + envFrom: + - configMapRef: + name: {{ include "speecht5.fullname" . }}-config + securityContext: + {{- if .Values.global.modelUseHostPath }} + {} + {{- else }} + {{- toYaml .Values.securityContext | nindent 12 }} + {{- end }} + image: huggingface/downloader:0.17.3 + command: ['sh', '-c'] + args: + - | + huggingface-cli download --cache-dir /data --token $(HF_TOKEN) {{ .Values.TTS_MODEL_PATH | quote }}; + huggingface-cli download --cache-dir /data --token $(HF_TOKEN) {{ .Values.VOCODER_MODEL| quote }}; + {{- if .Values.global.modelUseHostPath }} + chown -R 1000 /data/models--{{ replace "/" "--" .Values.TTS_MODEL_PATH }}; + chown -R 1000 /data/models--{{ replace "/" "--" .Values.VOCODER_MODEL }} + {{- end }} + volumeMounts: + - mountPath: /data + name: model-volume + {{- end }} containers: - name: {{ .Release.Name }} envFrom: diff --git a/helm-charts/common/speecht5/values.yaml b/helm-charts/common/speecht5/values.yaml index 96a8e452a..34822ae49 100644 --- a/helm-charts/common/speecht5/values.yaml +++ b/helm-charts/common/speecht5/values.yaml @@ -8,7 +8,7 @@ replicaCount: 1 TTS_MODEL_PATH: "microsoft/speecht5_tts" -# VOCODE_MODEL: "microsoft/speecht5_hifigan" +VOCODER_MODEL: "microsoft/speecht5_hifigan" image: repository: opea/speecht5 @@ -85,6 +85,7 @@ global: http_proxy: "" https_proxy: "" no_proxy: "" + HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here" # Choose where to save your downloaded models # Set modelUseHostPath for local directory, this is good for one node test. Example: diff --git a/helm-charts/common/tei/templates/configmap.yaml b/helm-charts/common/tei/templates/configmap.yaml index 6b40613a1..7f7c25301 100644 --- a/helm-charts/common/tei/templates/configmap.yaml +++ b/helm-charts/common/tei/templates/configmap.yaml @@ -32,3 +32,4 @@ data: {{- if .Values.MAX_WARMUP_SEQUENCE_LENGTH }} MAX_WARMUP_SEQUENCE_LENGTH: {{ .Values.MAX_WARMUP_SEQUENCE_LENGTH | quote }} {{- end }} + HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}} diff --git a/helm-charts/common/tei/templates/deployment.yaml b/helm-charts/common/tei/templates/deployment.yaml index 442847df5..44ca79876 100644 --- a/helm-charts/common/tei/templates/deployment.yaml +++ b/helm-charts/common/tei/templates/deployment.yaml @@ -30,6 +30,30 @@ spec: {{- end }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if not (hasPrefix "/data/" .Values.EMBEDDING_MODEL_ID) }} + initContainers: + - name: model-downloader + envFrom: + - configMapRef: + name: {{ include "tei.fullname" . }}-config + securityContext: + {{- if .Values.global.modelUseHostPath }} + {} + {{- else }} + {{- toYaml .Values.securityContext | nindent 12 }} + {{- end }} + image: huggingface/downloader:0.17.3 + command: ['huggingface-cli', 'download'] + args: + - "--cache-dir" + - "/data" + - "--token" + - $(HF_TOKEN) + - $(MODEL_ID) + volumeMounts: + - mountPath: /data + name: model-volume + {{- end }} containers: - name: {{ .Chart.Name }} envFrom: diff --git a/helm-charts/common/tei/values.yaml b/helm-charts/common/tei/values.yaml index 44460a359..88f9651bd 100644 --- a/helm-charts/common/tei/values.yaml +++ b/helm-charts/common/tei/values.yaml @@ -99,6 +99,7 @@ global: http_proxy: "" https_proxy: "" no_proxy: "" + HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here" # Choose where to save your downloaded models # Set modelUseHostPath for local directory, this is good for one node test. Example: diff --git a/helm-charts/common/teirerank/templates/configmap.yaml b/helm-charts/common/teirerank/templates/configmap.yaml index f8c25bc0b..e1a047eb4 100644 --- a/helm-charts/common/teirerank/templates/configmap.yaml +++ b/helm-charts/common/teirerank/templates/configmap.yaml @@ -32,3 +32,4 @@ data: {{- if .Values.MAX_WARMUP_SEQUENCE_LENGTH }} MAX_WARMUP_SEQUENCE_LENGTH: {{ .Values.MAX_WARMUP_SEQUENCE_LENGTH | quote }} {{- end }} + HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}} diff --git a/helm-charts/common/teirerank/templates/deployment.yaml b/helm-charts/common/teirerank/templates/deployment.yaml index 2d0ec9992..b1546e64d 100644 --- a/helm-charts/common/teirerank/templates/deployment.yaml +++ b/helm-charts/common/teirerank/templates/deployment.yaml @@ -30,6 +30,30 @@ spec: {{- end }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if not (hasPrefix "/data/" .Values.RERANK_MODEL_ID) }} + initContainers: + - name: model-downloader + envFrom: + - configMapRef: + name: {{ include "teirerank.fullname" . }}-config + securityContext: + {{- if .Values.global.modelUseHostPath }} + {} + {{- else }} + {{- toYaml .Values.securityContext | nindent 12 }} + {{- end }} + image: huggingface/downloader:0.17.3 + command: ['huggingface-cli', 'download'] + args: + - "--cache-dir" + - "/data" + - "--token" + - $(HF_TOKEN) + - $(MODEL_ID) + volumeMounts: + - mountPath: /data + name: model-volume + {{- end }} containers: - name: {{ .Chart.Name }} envFrom: diff --git a/helm-charts/common/teirerank/values.yaml b/helm-charts/common/teirerank/values.yaml index 4d0dedf29..63b1efebb 100644 --- a/helm-charts/common/teirerank/values.yaml +++ b/helm-charts/common/teirerank/values.yaml @@ -99,6 +99,7 @@ global: http_proxy: "" https_proxy: "" no_proxy: "" + HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here" # Choose where to save your downloaded models # Set modelUseHostPath for local directory, this is good for one node test. Example: diff --git a/helm-charts/common/tgi/templates/deployment.yaml b/helm-charts/common/tgi/templates/deployment.yaml index 04436e613..57f5b2194 100644 --- a/helm-charts/common/tgi/templates/deployment.yaml +++ b/helm-charts/common/tgi/templates/deployment.yaml @@ -30,6 +30,30 @@ spec: {{- end }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if not (hasPrefix "/data/" .Values.LLM_MODEL_ID) }} + initContainers: + - name: model-downloader + envFrom: + - configMapRef: + name: {{ include "tgi.fullname" . }}-config + securityContext: + {{- if .Values.global.modelUseHostPath }} + {} + {{- else }} + {{- toYaml .Values.securityContext | nindent 12 }} + {{- end }} + image: huggingface/downloader:0.17.3 + command: ['huggingface-cli', 'download'] + args: + - "--cache-dir" + - "/data" + - "--token" + - $(HF_TOKEN) + - $(MODEL_ID) + volumeMounts: + - mountPath: /data + name: model-volume + {{- end }} containers: - name: {{ .Chart.Name }} envFrom: diff --git a/helm-charts/common/vllm/templates/deployment.yaml b/helm-charts/common/vllm/templates/deployment.yaml index 66b5e3722..5783ad8f9 100644 --- a/helm-charts/common/vllm/templates/deployment.yaml +++ b/helm-charts/common/vllm/templates/deployment.yaml @@ -27,6 +27,30 @@ spec: {{- end }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if not (hasPrefix "/data/" .Values.LLM_MODEL_ID) }} + initContainers: + - name: model-downloader + envFrom: + - configMapRef: + name: {{ include "vllm.fullname" . }}-config + securityContext: + {{- if .Values.global.modelUseHostPath }} + {} + {{- else }} + {{- toYaml .Values.securityContext | nindent 12 }} + {{- end }} + image: huggingface/downloader:0.17.3 + command: ['huggingface-cli', 'download'] + args: + - "--cache-dir" + - "/data" + - "--token" + - $(HF_TOKEN) + - {{ .Values.LLM_MODEL_ID | quote }} + volumeMounts: + - mountPath: /data + name: model-volume + {{- end }} containers: - name: {{ .Chart.Name }} envFrom: diff --git a/helm-charts/common/whisper/templates/configmap.yaml b/helm-charts/common/whisper/templates/configmap.yaml index 2e27dea5d..39ab3db4e 100644 --- a/helm-charts/common/whisper/templates/configmap.yaml +++ b/helm-charts/common/whisper/templates/configmap.yaml @@ -18,3 +18,4 @@ data: HF_ENDPOINT: {{ .Values.global.HF_ENDPOINT | quote}} {{- end }} HUGGINGFACE_HUB_CACHE: "/data" + HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}} diff --git a/helm-charts/common/whisper/templates/deployment.yaml b/helm-charts/common/whisper/templates/deployment.yaml index c5d79899c..c72653e10 100644 --- a/helm-charts/common/whisper/templates/deployment.yaml +++ b/helm-charts/common/whisper/templates/deployment.yaml @@ -27,6 +27,30 @@ spec: {{- end }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if not (hasPrefix "/data/" .Values.ASR_MODEL_PATH) }} + initContainers: + - name: model-downloader + envFrom: + - configMapRef: + name: {{ include "whisper.fullname" . }}-config + securityContext: + {{- if .Values.global.modelUseHostPath }} + {} + {{- else }} + {{- toYaml .Values.securityContext | nindent 12 }} + {{- end }} + image: huggingface/downloader:0.17.3 + command: ['sh', '-c'] + args: + - | + huggingface-cli download --cache-dir /data --token $(HF_TOKEN) {{ .Values.ASR_MODEL_PATH | quote }}; + {{- if .Values.global.modelUseHostPath }} + chown -R 1000 /data/models--{{ replace "/" "--" .Values.ASR_MODEL_PATH }} + {{- end }} + volumeMounts: + - mountPath: /data + name: model-volume + {{- end }} containers: - name: {{ .Release.Name }} envFrom: diff --git a/helm-charts/common/whisper/values.yaml b/helm-charts/common/whisper/values.yaml index c368fe669..e3c62897c 100644 --- a/helm-charts/common/whisper/values.yaml +++ b/helm-charts/common/whisper/values.yaml @@ -84,6 +84,7 @@ global: http_proxy: "" https_proxy: "" no_proxy: "" + HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here" # Choose where to save your downloaded models # Set modelUseHostPath for local directory, this is good for one node test. Example: