fix(backend): Update compiler with new test sdk and fix dag publisher…

… execution id (#1425) * update compiler with new test sdk and fix dag publisher execution id * update publisher getParentDagID logic
kubeflow · Dec 7, 2023 · 552c477 · 552c477
1 parent d8ae28a
commit 552c477
Show file tree

Hide file tree

Showing 9 changed files with 52 additions and 54 deletions.
diff --git a/backend/src/v2/compiler/tektoncompiler/dag.go b/backend/src/v2/compiler/tektoncompiler/dag.go
@@ -371,14 +371,12 @@ func (i *pubDagDriverInputs) getDagType() string {
 	return "DAG_PUB"
 }
 
+// pubDagDriverInputs getParentDagID returns the parent node of the DAG publisher
+// which should always be the DAG driver DAG ID. However, exit handler doesn't
+// have driver so it's point to the root DAG ID instead.
 func (i *pubDagDriverInputs) getParentDagID(isExitHandler bool) string {
-	if i.parentDagID == "" {
-		return "0"
-	}
 	if isExitHandler && i.parentDagID == compiler.RootComponentName {
 		return fmt.Sprintf("$(params.%s)", paramParentDagID)
-	} else if i.inLoopDag {
-		return fmt.Sprintf("$(params.%s)", paramParentDagID)
 	} else {
 		return taskOutputParameter(getDAGDriverTaskName(i.parentDagID), paramExecutionID)
 	}

diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/condition_ir.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/condition_ir.yaml
@@ -98,7 +98,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.12'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -127,7 +127,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.12'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -156,7 +156,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.12'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -183,7 +183,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.12'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -210,7 +210,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.12'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -285,4 +285,4 @@ pipelineSpec:
           isOptional: true
           parameterType: STRING
   schemaVersion: 2.1.0
-  sdkVersion: kfp-2.0.0-beta.12
+  sdkVersion: kfp-2.0.1
diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml
@@ -44,7 +44,7 @@ spec:
                 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif
                   ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||
                   python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
-                  python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.0-beta.6''
+                  python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.1''
                   \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp
                   -d)\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\npython3
                   -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
@@ -97,7 +97,7 @@ spec:
                             python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
                         fi
 
-                        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+                        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1' && "$0" "$@"
                       - sh
                       - -ec
                       - |
@@ -205,7 +205,7 @@ spec:
         value: '{"args":["--executor_input","{{$}}","--function_to_execute","fail_op"],"command":["sh","-c","\nif
           ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3
           -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
-          python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.0-beta.6''
+          python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.1''
           \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\nprintf
           \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\npython3 -m
           kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
@@ -258,7 +258,7 @@ spec:
                     python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
                 fi
 
-                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1' && "$0" "$@"
               - sh
               - -ec
               - |
@@ -347,7 +347,7 @@ spec:
         value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif
           ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3
           -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
-          python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.0-beta.6''
+          python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.1''
           \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\nprintf
           \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\npython3 -m
           kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
@@ -400,7 +400,7 @@ spec:
                     python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
                 fi
 
-                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1' && "$0" "$@"
               - sh
               - -ec
               - |
@@ -512,8 +512,8 @@ spec:
       - name: mlmd-server-port
         value: "8080"
       runAfter:
-      - fail-op
       - print-op-2
+      - fail-op
       taskRef:
         apiVersion: custom.tekton.dev/v1alpha1
         kind: KFPTask

diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler_ir.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler_ir.yaml
@@ -62,7 +62,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -89,7 +89,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -116,7 +116,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -167,4 +167,4 @@ pipelineSpec:
           defaultValue: Hello World!
           parameterType: STRING
   schemaVersion: 2.1.0
-  sdkVersion: kfp-2.0.0-beta.6
+  sdkVersion: kfp-2.0.1
diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml
@@ -72,7 +72,7 @@ spec:
               - name: run-id
                 value: $(context.pipelineRun.uid)
               - name: dag-execution-id
-                value: $(params.parent-dag-id)
+                value: $(tasks.for-loop-2-loop-dag-driver.results.execution-id)
               - name: mlmd-server-address
                 value: metadata-grpc-service.kubeflow.svc.cluster.local
               - name: mlmd-server-port
@@ -98,7 +98,7 @@ spec:
                 value: '{"args":["--executor_input","{{$}}","--function_to_execute","concat_op"],"command":["sh","-c","\nif
                   ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||
                   python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
-                  python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.0-beta.15''
+                  python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.1''
                   \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp
                   -d)\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\npython3
                   -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
@@ -152,7 +152,7 @@ spec:
                             python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
                         fi
 
-                        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.15' && "$0" "$@"
+                        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1' && "$0" "$@"
                       - sh
                       - -ec
                       - |
@@ -239,7 +239,7 @@ spec:
                 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif
                   ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||
                   python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
-                  python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.0-beta.15''
+                  python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.1''
                   \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp
                   -d)\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\npython3
                   -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
@@ -295,7 +295,7 @@ spec:
                             python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
                         fi
 
-                        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.15' && "$0" "$@"
+                        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1' && "$0" "$@"
                       - sh
                       - -ec
                       - |
@@ -427,7 +427,7 @@ spec:
         value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif
           ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3
           -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
-          python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.0-beta.15''
+          python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.1''
           \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\nprintf
           \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\npython3 -m
           kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
@@ -480,7 +480,7 @@ spec:
                     python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
                 fi
 
-                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.15' && "$0" "$@"
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1' && "$0" "$@"
               - sh
               - -ec
               - |

diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/loop_static_ir.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/loop_static_ir.yaml
@@ -87,7 +87,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.15'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -114,7 +114,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.15'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -141,7 +141,7 @@ pipelineSpec:
           - -c
           - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
             \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.15'\
+            \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.1'\
             \ && \"$0\" \"$@\"\n"
           - sh
           - -ec
@@ -188,4 +188,4 @@ pipelineSpec:
           isOptional: true
           parameterType: STRING
   schemaVersion: 2.1.0
-  sdkVersion: kfp-2.0.0-beta.15
+  sdkVersion: kfp-2.0.1