From 75ab644250541089277c478229ae9835a7d11988 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Wed, 17 Jan 2024 00:53:54 -0800 Subject: [PATCH] feat(backend): update pod metadata test example (#1450) --- .../v2/compiler/tektoncompiler/tekton_test.go | 6 +- .../tektoncompiler/testdata/pod_metadata.yaml | 187 ++++++++++++++++++ .../testdata/create_pod_metadata.json | 22 +++ backend/third_party_licenses/driver.csv | 2 +- go.mod | 6 +- go.sum | 6 +- 6 files changed, 220 insertions(+), 9 deletions(-) create mode 100644 backend/src/v2/compiler/tektoncompiler/testdata/pod_metadata.yaml create mode 100644 backend/src/v2/compiler/testdata/create_pod_metadata.json diff --git a/backend/src/v2/compiler/tektoncompiler/tekton_test.go b/backend/src/v2/compiler/tektoncompiler/tekton_test.go index 3d81d53767..95253621c6 100644 --- a/backend/src/v2/compiler/tektoncompiler/tekton_test.go +++ b/backend/src/v2/compiler/tektoncompiler/tekton_test.go @@ -51,9 +51,9 @@ func Test_tekton_compiler(t *testing.T) { tektonYAMLPath: "testdata/importer.yaml", }, { - jobPath: "../testdata/importer.json", - platformSpecPath: "", - tektonYAMLPath: "testdata/importer.yaml", + jobPath: "../testdata/hello_world.json", + platformSpecPath: "../testdata/create_pod_metadata.json", + tektonYAMLPath: "testdata/pod_metadata.yaml", }, } for _, tt := range tests { diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/pod_metadata.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/pod_metadata.yaml new file mode 100644 index 0000000000..58cf1d2011 --- /dev/null +++ b/backend/src/v2/compiler/tektoncompiler/testdata/pod_metadata.yaml @@ -0,0 +1,187 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + pipelines.kubeflow.org/v2_pipeline: "true" + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// + creationTimestamp: null + generateName: hello-world- + labels: + pipelines.kubeflow.org/v2_component: "true" +spec: + pipelineSpec: + tasks: + - name: hello-world + params: + - name: type + value: CONTAINER + - name: pipeline-name + value: namespace/n1/pipeline/hello-world + - name: run-id + value: $(context.pipelineRun.uid) + - name: dag-execution-id + value: $(tasks.root-system-dag-driver.results.execution-id) + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' + - name: container + value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf + \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def + hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser + = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", + dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args + = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' + - name: iteration-index + value: "" + - name: kubernetes-config + value: '{"podMetadata":{"annotations":{"experiment_id":"234567","run_id":"123456"},"labels":{"kubeflow.com/common":"test","kubeflow.com/kfp":"pipeline-node"}}}' + - name: mlmd-server-address + value: metadata-grpc-service.kubeflow.svc.cluster.local + - name: mlmd-server-port + value: "8080" + - name: component + value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' + taskSpec: + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + metadata: + annotations: + pipelines.kubeflow.org/v2_pipeline: "true" + labels: + pipelines.kubeflow.org/v2_component: "true" + spec: + taskSpec: + params: + - name: executor-input + type: string + - name: execution-id + type: string + - name: run-id + type: string + - name: component + type: string + steps: + - command: + - launcher-v2 + - --copy + - /tekton/home/launch + computeResources: {} + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e + imagePullPolicy: Always + name: kfp-launcher + - args: + - sh + - -ec + - | + program_path=$(mktemp) + printf "%s" "$0" > "$program_path" + python3 -u "$program_path" "$@" + - | + def hello_world(text): + print(text) + return text + + import argparse + _parser = argparse.ArgumentParser(prog='Hello world', description='') + _parser.add_argument("--text", dest="text", type=str, required=True, default=argparse.SUPPRESS) + _parsed_args = vars(_parser.parse_args()) + + _outputs = hello_world(**_parsed_args) + - --text + - '{{$.inputs.parameters[''text'']}}' + command: + - /tekton/home/launch + - --pipeline_name + - namespace/n1/pipeline/hello-world + - --run_id + - $(params.run-id) + - --execution_id + - $(params.execution-id) + - --executor_input + - $(params.executor-input) + - --component_spec + - $(params.component) + - --pod_name + - $(KFP_POD_NAME) + - --pod_uid + - $(KFP_POD_UID) + - --mlmd_server_address + - $(METADATA_GRPC_SERVICE_HOST) + - --mlmd_server_port + - $(METADATA_GRPC_SERVICE_PORT) + - -- + computeResources: {} + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: METADATA_GRPC_SERVICE_HOST + value: metadata-grpc-service.kubeflow.svc.cluster.local + - name: METADATA_GRPC_SERVICE_PORT + value: "8080" + - name: ML_PIPELINE_SERVICE_HOST + value: ml-pipeline.kubeflow.svc.cluster.local + - name: ML_PIPELINE_SERVICE_PORT_GRPC + value: "8887" + - name: MINIO_SERVICE_SERVICE_HOST + value: minio-service.kubeflow.svc.cluster.local + - name: MINIO_SERVICE_SERVICE_PORT + value: "9000" + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: python:3.7 + name: user-main + - name: root-system-dag-driver + params: + - name: type + value: ROOT_DAG + - name: pipeline-name + value: namespace/n1/pipeline/hello-world + - name: run-id + value: $(context.pipelineRun.uid) + - name: dag-execution-id + value: "0" + - name: component + value: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' + - name: task + value: "" + - name: runtime-config + value: '{"parameters":{"text":{"stringValue":"hi there"}}}' + - name: iteration-index + value: "-1" + - name: mlmd-server-address + value: metadata-grpc-service.kubeflow.svc.cluster.local + - name: mlmd-server-port + value: "8080" + taskRef: + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + - name: root-system-dag-pub-driver + params: + - name: type + value: DAG_PUB + - name: pipeline-name + value: namespace/n1/pipeline/hello-world + - name: run-id + value: $(context.pipelineRun.uid) + - name: dag-execution-id + value: $(tasks.root-system-dag-driver.results.execution-id) + - name: mlmd-server-address + value: metadata-grpc-service.kubeflow.svc.cluster.local + - name: mlmd-server-port + value: "8080" + runAfter: + - hello-world + taskRef: + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + taskRunTemplate: {} +status: {} diff --git a/backend/src/v2/compiler/testdata/create_pod_metadata.json b/backend/src/v2/compiler/testdata/create_pod_metadata.json new file mode 100644 index 0000000000..246d3b1dd2 --- /dev/null +++ b/backend/src/v2/compiler/testdata/create_pod_metadata.json @@ -0,0 +1,22 @@ +{ + "platforms": { + "kubernetes": { + "deploymentSpec": { + "executors": { + "exec-hello-world": { + "podMetadata": { + "annotations": { + "run_id": "123456", + "experiment_id": "234567" + }, + "labels": { + "kubeflow.com/kfp": "pipeline-node", + "kubeflow.com/common": "test" + } + } + } + } + } + } + } + } diff --git a/backend/third_party_licenses/driver.csv b/backend/third_party_licenses/driver.csv index 2da1931c16..3444025b37 100644 --- a/backend/third_party_licenses/driver.csv +++ b/backend/third_party_licenses/driver.csv @@ -31,7 +31,7 @@ github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/lice github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT github.com/kubeflow/pipelines/api/v2alpha1/go,https://github.com/kubeflow/pipelines/blob/58ce09e07d03/api/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 -github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/58ce09e07d03/kubernetes_platform/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/aac4408237df/kubernetes_platform/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e1f0c010f800/third_party/ml-metadata/LICENSE,Apache-2.0 github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.7/LICENSE,MIT github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 diff --git a/go.mod b/go.mod index 1376b6aca8..90fb9c04bb 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-exithandler v0.0.0-20231127195001-a75d4b3711ff github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-kfptask v0.0.0-20231127195001-a75d4b3711ff github.com/kubeflow/pipelines/api v0.0.0-20231027040853-58ce09e07d03 - github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20231027040853-58ce09e07d03 + github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240111221413-aac4408237df github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800 github.com/lestrrat-go/strftime v1.0.4 github.com/mattn/go-sqlite3 v1.14.16 @@ -201,8 +201,8 @@ require ( ) replace ( - github.com/kubeflow/kfp-tekton/tekton-catalog/cache => ./tekton-catalog/cache - github.com/kubeflow/kfp-tekton/tekton-catalog/objectstore => ./tekton-catalog/objectstore + github.com/kubeflow/kfp-tekton/tekton-catalog/cache => github.com/kubeflow/kfp-tekton/tekton-catalog/cache v0.0.0-20231127195001-a75d4b3711ff + github.com/kubeflow/kfp-tekton/tekton-catalog/objectstore => github.com/kubeflow/kfp-tekton/tekton-catalog/objectstore v0.0.0-20231127195001-a75d4b3711ff k8s.io/api => k8s.io/api v0.25.9 k8s.io/apimachinery => k8s.io/apimachinery v0.26.5 k8s.io/client-go => k8s.io/client-go v0.25.9 diff --git a/go.sum b/go.sum index 363f3e237b..3deaa5ffd8 100644 --- a/go.sum +++ b/go.sum @@ -2180,6 +2180,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubeflow/kfp-tekton/tekton-catalog/cache v0.0.0-20231127195001-a75d4b3711ff/go.mod h1:LsG0iGcQwQ1y1WfkU7Pi9f5zlUJsQ7ja/vWWonVmkKY= +github.com/kubeflow/kfp-tekton/tekton-catalog/objectstore v0.0.0-20231127195001-a75d4b3711ff/go.mod h1:frzWuDDCi93/PWg3OOC16biM4s/y9NlwJQam80lqbKE= github.com/kubeflow/kfp-tekton/tekton-catalog/pipeline-loops v0.0.0-20231127195001-a75d4b3711ff h1:IDw72D1gjs5uW5ioxbPSBmWuQmId9bKmmfm7vZqg2Q0= github.com/kubeflow/kfp-tekton/tekton-catalog/pipeline-loops v0.0.0-20231127195001-a75d4b3711ff/go.mod h1:/A/oAAVtsZdKgp/sxB0QA9vuWNprwdq33qqFjsbgUKo= github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-exithandler v0.0.0-20231127195001-a75d4b3711ff h1:lmRqwrPNdXxY5Yl4uGLHJQhmvlazwhzLD1ZmYCcA6GU= @@ -2188,8 +2190,8 @@ github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-kfptask v0.0.0-202311271950 github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-kfptask v0.0.0-20231127195001-a75d4b3711ff/go.mod h1:lAFdPugzj3bcAXyN3+8y0NByidZ88zwGxMc+gdc8cHw= github.com/kubeflow/pipelines/api v0.0.0-20231027040853-58ce09e07d03 h1:reL3LbkRIozBkKSUYjtQFV2kVC1R4WHG9FrTClRT1FY= github.com/kubeflow/pipelines/api v0.0.0-20231027040853-58ce09e07d03/go.mod h1:T7TOQB36gGe97yUdfVAnYK5uuT0+uQbLNHDUHxYkmE4= -github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20231027040853-58ce09e07d03 h1:5GgKSLMNvwRfrLxKCrHvlVVcAo+bfrTZzbR+hOGHDIs= -github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20231027040853-58ce09e07d03/go.mod h1:CJkKr356RlpZP/gQRuHf3Myrn1qJtoUVe4EMCmtwarg= +github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240111221413-aac4408237df h1:hskr5e/bAKNi8Bc15lG9uZ28q36v/y8qbKkDDix9NPo= +github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240111221413-aac4408237df/go.mod h1:CJkKr356RlpZP/gQRuHf3Myrn1qJtoUVe4EMCmtwarg= github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800 h1:YAW+X9xCW8Yq5tQaBBQaLTNU9CJj8Nr7lx1+k66ZHJ0= github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800/go.mod h1:chIDffBaVQ/asNl1pTTdbAymYcuBKf8BR3YtSP+3FEU= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=