From 7e0dcbe307ef9ff36d31ae4a0747b5355aecca42 Mon Sep 17 00:00:00 2001 From: Elliot Gunton Date: Fri, 9 Feb 2024 14:27:42 +0000 Subject: [PATCH] Add template_level_volume example (#954) * Upstream example but only appears in docs so is not part of the usual collection * User in slack was asking how to recreate this example, see https://cloud-native.slack.com/archives/C03NRMD9KPY/p1707226500216859 Signed-off-by: Elliot Gunton --- .../workflows/template_level_volume.md | 153 ++++++++++++++++++ .../workflows/use-cases/fine_tune_llama.md | 98 +++++++---- examples/workflows/template-level-volume.yaml | 83 ++++++++++ examples/workflows/template_level_volume.py | 55 +++++++ .../workflows/use_cases/fine-tune-llama.yaml | 98 +++++++---- 5 files changed, 433 insertions(+), 54 deletions(-) create mode 100644 docs/examples/workflows/template_level_volume.md create mode 100644 examples/workflows/template-level-volume.yaml create mode 100644 examples/workflows/template_level_volume.py diff --git a/docs/examples/workflows/template_level_volume.md b/docs/examples/workflows/template_level_volume.md new file mode 100644 index 000000000..883e6758f --- /dev/null +++ b/docs/examples/workflows/template_level_volume.md @@ -0,0 +1,153 @@ +# Template Level Volume + + + +This is an upstream example that only appears in the docs. + +See https://argo-workflows.readthedocs.io/en/latest/walk-through/volumes/ + + +=== "Hera" + + ```python linenums="1" + from hera.workflows import ( + Container, + ExistingVolume, + Parameter, + Resource, + Step, + Steps, + Workflow, + ) + + with Workflow( + generate_name="template-level-volume-", + entrypoint="generate-and-use-volume", + ) as w: + generate_volume = Resource( + name="generate-volume", + inputs=[Parameter(name="pvc-size")], + outputs=[Parameter(name="pvc-name", value_from={"jsonPath": "{.metadata.name}"})], + action="create", + set_owner_reference=True, + manifest="""apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + generateName: pvc-example- + spec: + accessModes: ['ReadWriteOnce', 'ReadOnlyMany'] + resources: + requests: + storage: '{{inputs.parameters.pvc-size}}' + """, + ) + whalesay = Container( + name="whalesay", + inputs=[Parameter(name="pvc-name")], + volumes=[ExistingVolume(name="workdir", claim_name="{{inputs.parameters.pvc-name}}", mount_path="/mnt/vol")], + image="docker/whalesay:latest", + command=["sh", "-c"], + args=["echo generating message in volume; cowsay hello world | tee /mnt/vol/hello_world.txt"], + ) + print_message = Container( + name="print-message", + inputs=[Parameter(name="pvc-name")], + volumes=[ExistingVolume(name="workdir", claim_name="{{inputs.parameters.pvc-name}}", mount_path="/mnt/vol")], + image="alpine:latest", + command=["sh", "-c"], + args=["echo getting message from volume; find /mnt/vol; cat /mnt/vol/hello_world.txt"], + ) + with Steps(name="generate-and-use-volume") as s: + generate_vol_step: Step = generate_volume(name="generate-volume", arguments={"pvc-size": "1Gi"}) + whalesay(name="generate", arguments=generate_vol_step.get_parameter("pvc-name")) + print_message(name="print", arguments=generate_vol_step.get_parameter("pvc-name")) + ``` + +=== "YAML" + + ```yaml linenums="1" + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: template-level-volume- + spec: + entrypoint: generate-and-use-volume + templates: + - name: generate-and-use-volume + steps: + - - name: generate-volume + template: generate-volume + arguments: + parameters: + - name: pvc-size + # In a real-world example, this could be generated by a previous workflow step. + value: '1Gi' + - - name: generate + template: whalesay + arguments: + parameters: + - name: pvc-name + value: '{{steps.generate-volume.outputs.parameters.pvc-name}}' + - - name: print + template: print-message + arguments: + parameters: + - name: pvc-name + value: '{{steps.generate-volume.outputs.parameters.pvc-name}}' + + - name: generate-volume + inputs: + parameters: + - name: pvc-size + resource: + action: create + setOwnerReference: true + manifest: | + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + generateName: pvc-example- + spec: + accessModes: ['ReadWriteOnce', 'ReadOnlyMany'] + resources: + requests: + storage: '{{inputs.parameters.pvc-size}}' + outputs: + parameters: + - name: pvc-name + valueFrom: + jsonPath: '{.metadata.name}' + + - name: whalesay + inputs: + parameters: + - name: pvc-name + volumes: + - name: workdir + persistentVolumeClaim: + claimName: '{{inputs.parameters.pvc-name}}' + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["echo generating message in volume; cowsay hello world | tee /mnt/vol/hello_world.txt"] + volumeMounts: + - name: workdir + mountPath: /mnt/vol + + - name: print-message + inputs: + parameters: + - name: pvc-name + volumes: + - name: workdir + persistentVolumeClaim: + claimName: '{{inputs.parameters.pvc-name}}' + container: + image: alpine:latest + command: [sh, -c] + args: ["echo getting message from volume; find /mnt/vol; cat /mnt/vol/hello_world.txt"] + volumeMounts: + - name: workdir + mountPath: /mnt/vol + ``` + diff --git a/docs/examples/workflows/use-cases/fine_tune_llama.md b/docs/examples/workflows/use-cases/fine_tune_llama.md index 5a849b5cf..0c6bd1c6e 100644 --- a/docs/examples/workflows/use-cases/fine_tune_llama.md +++ b/docs/examples/workflows/use-cases/fine_tune_llama.md @@ -393,32 +393,65 @@ There are several implicit dependencies in this script: - name: create-ssd-storage-class resource: action: create - manifest: "\nkind: StorageClass\napiVersion: storage.k8s.io/v1\nmetadata:\n\ - \ name: ssd\nprovisioner: kubernetes.io/gce-pd\nvolumeBindingMode: WaitForFirstConsumer\n\ - parameters:\n type: pd-ssd\n" + manifest: |2 + + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: ssd + provisioner: kubernetes.io/gce-pd + volumeBindingMode: WaitForFirstConsumer + parameters: + type: pd-ssd - name: create-etcd-stateful-set resource: action: create - manifest: "\napiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n name: etcd\n\ - \ labels:\n app: etcd\nspec:\n serviceName: etcd\n selector:\n matchLabels:\n\ - \ app: etcd\n replicas: 3\n template:\n metadata:\n name: etcd\n\ - \ labels:\n app: etcd\n spec:\n containers:\n -\ - \ name: etcd\n image: quay.io/coreos/etcd:latest\n ports:\n\ - \ - containerPort: 2379\n name: client\n \ - \ - containerPort: 2380\n name: peer\n volumeMounts:\n\ - \ - name: data\n mountPath: /var/run/etcd\n \ - \ command:\n - /bin/sh\n - -c\n - |\n\ - \ PEERS=\"etcd-0=http://etcd-0.etcd:2380,etcd-1=http://etcd-1.etcd:2380,etcd-2=http://etcd-2.etcd:2380\"\ - \n exec etcd --name ${HOSTNAME} --listen-peer-urls\ - \ http://0.0.0.0:2380 --listen-client-urls http://0.0.0.0:2379\ - \ --advertise-client-urls http://${HOSTNAME}.etcd:2379 \ - \ --initial-advertise-peer-urls http://${HOSTNAME}:2380 \ - \ --initial-cluster-token etcd-cluster-1 --initial-cluster\ - \ ${PEERS} --initial-cluster-state new --data-dir\ - \ /var/run/etcd/default.etcd\n volumeClaimTemplates:\n - metadata:\n \ - \ name: data\n spec:\n storageClassName: ssd\n accessModes:\ - \ [\"ReadWriteOnce\"]\n resources:\n requests:\n \ - \ storage: 1Gi" + manifest: |2- + + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: etcd + labels: + app: etcd + spec: + serviceName: etcd + selector: + matchLabels: + app: etcd + replicas: 3 + template: + metadata: + name: etcd + labels: + app: etcd + spec: + containers: + - name: etcd + image: quay.io/coreos/etcd:latest + ports: + - containerPort: 2379 + name: client + - containerPort: 2380 + name: peer + volumeMounts: + - name: data + mountPath: /var/run/etcd + command: + - /bin/sh + - -c + - | + PEERS="etcd-0=http://etcd-0.etcd:2380,etcd-1=http://etcd-1.etcd:2380,etcd-2=http://etcd-2.etcd:2380" + exec etcd --name ${HOSTNAME} --listen-peer-urls http://0.0.0.0:2380 --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://${HOSTNAME}.etcd:2379 --initial-advertise-peer-urls http://${HOSTNAME}:2380 --initial-cluster-token etcd-cluster-1 --initial-cluster ${PEERS} --initial-cluster-state new --data-dir /var/run/etcd/default.etcd + volumeClaimTemplates: + - metadata: + name: data + spec: + storageClassName: ssd + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi - name: create-etcd-load-balancer outputs: parameters: @@ -427,10 +460,21 @@ There are several implicit dependencies in this script: jsonPath: metadata.name resource: action: create - manifest: "\napiVersion: v1\nkind: Service\nmetadata:\n name: etcd-client\n\ - spec:\n type: LoadBalancer\n ports:\n - name: etcd-client\n port:\ - \ 2379\n protocol: TCP\n targetPort: 2379\n selector:\n app:\ - \ etcd" + manifest: |2- + + apiVersion: v1 + kind: Service + metadata: + name: etcd-client + spec: + type: LoadBalancer + ports: + - name: etcd-client + port: 2379 + protocol: TCP + targetPort: 2379 + selector: + app: etcd - container: args: - etcd_ip=""; while [ -z $etcd_ip ]; do echo "Waiting for end point..."; etcd_ip=$(kubectl diff --git a/examples/workflows/template-level-volume.yaml b/examples/workflows/template-level-volume.yaml new file mode 100644 index 000000000..5032067bb --- /dev/null +++ b/examples/workflows/template-level-volume.yaml @@ -0,0 +1,83 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: template-level-volume- +spec: + entrypoint: generate-and-use-volume + templates: + - name: generate-and-use-volume + steps: + - - name: generate-volume + template: generate-volume + arguments: + parameters: + - name: pvc-size + # In a real-world example, this could be generated by a previous workflow step. + value: '1Gi' + - - name: generate + template: whalesay + arguments: + parameters: + - name: pvc-name + value: '{{steps.generate-volume.outputs.parameters.pvc-name}}' + - - name: print + template: print-message + arguments: + parameters: + - name: pvc-name + value: '{{steps.generate-volume.outputs.parameters.pvc-name}}' + + - name: generate-volume + inputs: + parameters: + - name: pvc-size + resource: + action: create + setOwnerReference: true + manifest: | + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + generateName: pvc-example- + spec: + accessModes: ['ReadWriteOnce', 'ReadOnlyMany'] + resources: + requests: + storage: '{{inputs.parameters.pvc-size}}' + outputs: + parameters: + - name: pvc-name + valueFrom: + jsonPath: '{.metadata.name}' + + - name: whalesay + inputs: + parameters: + - name: pvc-name + volumes: + - name: workdir + persistentVolumeClaim: + claimName: '{{inputs.parameters.pvc-name}}' + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["echo generating message in volume; cowsay hello world | tee /mnt/vol/hello_world.txt"] + volumeMounts: + - name: workdir + mountPath: /mnt/vol + + - name: print-message + inputs: + parameters: + - name: pvc-name + volumes: + - name: workdir + persistentVolumeClaim: + claimName: '{{inputs.parameters.pvc-name}}' + container: + image: alpine:latest + command: [sh, -c] + args: ["echo getting message from volume; find /mnt/vol; cat /mnt/vol/hello_world.txt"] + volumeMounts: + - name: workdir + mountPath: /mnt/vol diff --git a/examples/workflows/template_level_volume.py b/examples/workflows/template_level_volume.py new file mode 100644 index 000000000..0d1333730 --- /dev/null +++ b/examples/workflows/template_level_volume.py @@ -0,0 +1,55 @@ +"""This is an upstream example that only appears in the docs. + +See https://argo-workflows.readthedocs.io/en/latest/walk-through/volumes/""" + +from hera.workflows import ( + Container, + ExistingVolume, + Parameter, + Resource, + Step, + Steps, + Workflow, +) + +with Workflow( + generate_name="template-level-volume-", + entrypoint="generate-and-use-volume", +) as w: + generate_volume = Resource( + name="generate-volume", + inputs=[Parameter(name="pvc-size")], + outputs=[Parameter(name="pvc-name", value_from={"jsonPath": "{.metadata.name}"})], + action="create", + set_owner_reference=True, + manifest="""apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + generateName: pvc-example- +spec: + accessModes: ['ReadWriteOnce', 'ReadOnlyMany'] + resources: + requests: + storage: '{{inputs.parameters.pvc-size}}' +""", + ) + whalesay = Container( + name="whalesay", + inputs=[Parameter(name="pvc-name")], + volumes=[ExistingVolume(name="workdir", claim_name="{{inputs.parameters.pvc-name}}", mount_path="/mnt/vol")], + image="docker/whalesay:latest", + command=["sh", "-c"], + args=["echo generating message in volume; cowsay hello world | tee /mnt/vol/hello_world.txt"], + ) + print_message = Container( + name="print-message", + inputs=[Parameter(name="pvc-name")], + volumes=[ExistingVolume(name="workdir", claim_name="{{inputs.parameters.pvc-name}}", mount_path="/mnt/vol")], + image="alpine:latest", + command=["sh", "-c"], + args=["echo getting message from volume; find /mnt/vol; cat /mnt/vol/hello_world.txt"], + ) + with Steps(name="generate-and-use-volume") as s: + generate_vol_step: Step = generate_volume(name="generate-volume", arguments={"pvc-size": "1Gi"}) + whalesay(name="generate", arguments=generate_vol_step.get_parameter("pvc-name")) + print_message(name="print", arguments=generate_vol_step.get_parameter("pvc-name")) diff --git a/examples/workflows/use_cases/fine-tune-llama.yaml b/examples/workflows/use_cases/fine-tune-llama.yaml index 3cc56530b..a44b03d97 100644 --- a/examples/workflows/use_cases/fine-tune-llama.yaml +++ b/examples/workflows/use_cases/fine-tune-llama.yaml @@ -79,32 +79,65 @@ spec: - name: create-ssd-storage-class resource: action: create - manifest: "\nkind: StorageClass\napiVersion: storage.k8s.io/v1\nmetadata:\n\ - \ name: ssd\nprovisioner: kubernetes.io/gce-pd\nvolumeBindingMode: WaitForFirstConsumer\n\ - parameters:\n type: pd-ssd\n" + manifest: |2 + + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: ssd + provisioner: kubernetes.io/gce-pd + volumeBindingMode: WaitForFirstConsumer + parameters: + type: pd-ssd - name: create-etcd-stateful-set resource: action: create - manifest: "\napiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n name: etcd\n\ - \ labels:\n app: etcd\nspec:\n serviceName: etcd\n selector:\n matchLabels:\n\ - \ app: etcd\n replicas: 3\n template:\n metadata:\n name: etcd\n\ - \ labels:\n app: etcd\n spec:\n containers:\n -\ - \ name: etcd\n image: quay.io/coreos/etcd:latest\n ports:\n\ - \ - containerPort: 2379\n name: client\n \ - \ - containerPort: 2380\n name: peer\n volumeMounts:\n\ - \ - name: data\n mountPath: /var/run/etcd\n \ - \ command:\n - /bin/sh\n - -c\n - |\n\ - \ PEERS=\"etcd-0=http://etcd-0.etcd:2380,etcd-1=http://etcd-1.etcd:2380,etcd-2=http://etcd-2.etcd:2380\"\ - \n exec etcd --name ${HOSTNAME} --listen-peer-urls\ - \ http://0.0.0.0:2380 --listen-client-urls http://0.0.0.0:2379\ - \ --advertise-client-urls http://${HOSTNAME}.etcd:2379 \ - \ --initial-advertise-peer-urls http://${HOSTNAME}:2380 \ - \ --initial-cluster-token etcd-cluster-1 --initial-cluster\ - \ ${PEERS} --initial-cluster-state new --data-dir\ - \ /var/run/etcd/default.etcd\n volumeClaimTemplates:\n - metadata:\n \ - \ name: data\n spec:\n storageClassName: ssd\n accessModes:\ - \ [\"ReadWriteOnce\"]\n resources:\n requests:\n \ - \ storage: 1Gi" + manifest: |2- + + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: etcd + labels: + app: etcd + spec: + serviceName: etcd + selector: + matchLabels: + app: etcd + replicas: 3 + template: + metadata: + name: etcd + labels: + app: etcd + spec: + containers: + - name: etcd + image: quay.io/coreos/etcd:latest + ports: + - containerPort: 2379 + name: client + - containerPort: 2380 + name: peer + volumeMounts: + - name: data + mountPath: /var/run/etcd + command: + - /bin/sh + - -c + - | + PEERS="etcd-0=http://etcd-0.etcd:2380,etcd-1=http://etcd-1.etcd:2380,etcd-2=http://etcd-2.etcd:2380" + exec etcd --name ${HOSTNAME} --listen-peer-urls http://0.0.0.0:2380 --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://${HOSTNAME}.etcd:2379 --initial-advertise-peer-urls http://${HOSTNAME}:2380 --initial-cluster-token etcd-cluster-1 --initial-cluster ${PEERS} --initial-cluster-state new --data-dir /var/run/etcd/default.etcd + volumeClaimTemplates: + - metadata: + name: data + spec: + storageClassName: ssd + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi - name: create-etcd-load-balancer outputs: parameters: @@ -113,10 +146,21 @@ spec: jsonPath: metadata.name resource: action: create - manifest: "\napiVersion: v1\nkind: Service\nmetadata:\n name: etcd-client\n\ - spec:\n type: LoadBalancer\n ports:\n - name: etcd-client\n port:\ - \ 2379\n protocol: TCP\n targetPort: 2379\n selector:\n app:\ - \ etcd" + manifest: |2- + + apiVersion: v1 + kind: Service + metadata: + name: etcd-client + spec: + type: LoadBalancer + ports: + - name: etcd-client + port: 2379 + protocol: TCP + targetPort: 2379 + selector: + app: etcd - container: args: - etcd_ip=""; while [ -z $etcd_ip ]; do echo "Waiting for end point..."; etcd_ip=$(kubectl