From 87bff179b80d1e54e39faf7ca7fb97eec835ac78 Mon Sep 17 00:00:00 2001 From: David Wood Date: Mon, 7 Oct 2024 08:02:25 -0400 Subject: [PATCH] Fix kfp workflows which were created with a bad template (#672) * fix typo in kfp workflows Signed-off-by: David Wood * fix another typo in kfp workflows Signed-off-by: David Wood * innocuous change in .make.workflows to trigger kfp tests Signed-off-by: David Wood * fix missing path in workflow templates Signed-off-by: David Wood * fix repo_level_order_wf.py default float value of str type Signed-off-by: David Wood * workflow generation fixes and kfp workflow to test for kfp_ray/Makefile Signed-off-by: David Wood * disable kfp cicd for repo_level_ordering and text_encoder Signed-off-by: David Wood * fix testing for makefile in kfp workflows Signed-off-by: David Wood * remove kfp_support_list testing from transform's kfp git workflow Signed-off-by: David Wood * disable kfp cicd on license_select Signed-off-by: David Wood --------- Signed-off-by: David Wood --- .github/workflows/Makefile | 23 +++++++++++++------ .../workflows/test-code-code2parquet-kfp.yml | 20 ++++++++-------- .../workflows/test-code-code_quality-kfp.yml | 20 ++++++++-------- .../test-code-header_cleanser-kfp.yml | 20 ++++++++-------- .../test-code-license_select-kfp.yml | 20 ++++++++-------- .github/workflows/test-code-malware-kfp.yml | 20 ++++++++-------- .../test-code-proglang_select-kfp.yml | 20 ++++++++-------- .../test-code-repo_level_ordering-kfp.yml | 20 ++++++++-------- .github/workflows/test-kfp-transform.template | 20 ++++++++-------- .../test-language-doc_quality-kfp.yml | 20 ++++++++-------- .../workflows/test-language-lang_id-kfp.yml | 20 ++++++++-------- .../test-language-text_encoder-kfp.yml | 20 ++++++++-------- .../workflows/test-universal-doc_id-kfp.yml | 20 ++++++++-------- .../workflows/test-universal-ededup-kfp.yml | 20 ++++++++-------- .../workflows/test-universal-fdedup-kfp.yml | 20 ++++++++-------- .../workflows/test-universal-filter-kfp.yml | 20 ++++++++-------- .github/workflows/test-universal-noop-kfp.yml | 20 ++++++++-------- .../workflows/test-universal-profiler-kfp.yml | 20 ++++++++-------- .../workflows/test-universal-resize-kfp.yml | 20 ++++++++-------- .../test-universal-tokenization-kfp.yml | 20 ++++++++-------- transforms/.make.workflows | 2 +- .../{Makefile => Makefile.disable-cicd} | 0 .../{Makefile => Makefile.disable-cicd} | 0 .../kfp_ray/repo_level_order_wf.py | 17 ++++++++++---- .../{Makefile => Makefile.disable-cicd} | 0 25 files changed, 220 insertions(+), 202 deletions(-) rename transforms/code/license_select/kfp_ray/{Makefile => Makefile.disable-cicd} (100%) rename transforms/code/repo_level_ordering/kfp_ray/{Makefile => Makefile.disable-cicd} (100%) rename transforms/language/text_encoder/kfp_ray/{Makefile => Makefile.disable-cicd} (100%) diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index 751b3201f..8d0c039dc 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -10,10 +10,13 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco # A list that holds transforms that should not be tested with KFP transform-tests: + -rm -rf test-universal-*.yml $(MAKE) TRANSFORM_SUBDIR=universal .transform-tests $(MAKE) TRANSFORM_SUBDIR=universal .transform-kfp-tests + -rm -rf test-language-*.yml $(MAKE) TRANSFORM_SUBDIR=language .transform-tests $(MAKE) TRANSFORM_SUBDIR=language .transform-kfp-tests + -rm -rf test-code-*.yml $(MAKE) TRANSFORM_SUBDIR=code .transform-tests $(MAKE) TRANSFORM_SUBDIR=code .transform-kfp-tests @@ -21,28 +24,34 @@ transform-tests: # TRANSFORM_SUBDIR transforms subdirectory (such as universal) .transform-tests: @for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \ - dir=$$(basename $$i); \ - yml=test-$(TRANSFORM_SUBDIR)-$$dir.yml; \ - echo Generating $$yml; \ - cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \ + z=$$(ls $$i/Makefile*); \ + if [ ! -z "$$z" ]; then \ + dir=$$(basename $$i); \ + yml=test-$(TRANSFORM_SUBDIR)-$$dir.yml; \ + echo Generating $$yml; \ + cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \ + fi; \ done .transform-kfp-tests: @KFP_BLACK_LIST=$$(cd ../..; bash scripts/check-workflows.sh -show-kfp-black-list); \ for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \ + z=$$(ls $$i/Makefile*); \ + if [ ! -z "$$z" ]; then \ dir=$$(basename $$i); \ yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml; \ if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ]; then \ - echo No kfp_ray directory for $$dir. Skipping generation of $$yml; \ + echo No kfp_ray directory for $$dir. Skipping generation of $$yml; \ continue; \ fi; \ z=$$(echo $${KFP_BLACK_LIST} | grep $$dir); \ if [ ! -z "$$z" ]; then \ - echo $$dir is black listed. Skipping generation of $$yml; \ + echo $$dir is black listed. Skipping generation of $$yml; \ continue; \ fi; \ - echo Generating $$yml; \ + echo Generating $$yml; \ cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \ + fi; \ done diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml index 6de24d4b0..9b0a92042 100644 --- a/.github/workflows/test-code-code2parquet-kfp.yml +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/code2parquet/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/code2parquet/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/code/code2parquet timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/code2parquet workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/code2parquet workflow-test echo "Run transforms/code/code2parquet completed" else - echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/code/code2parquet timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/code2parquet workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/code2parquet workflow-test echo "Run transforms/code/code2parquet completed" else - echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml index 2e22c04a9..504923b6b 100644 --- a/.github/workflows/test-code-code_quality-kfp.yml +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/code_quality/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/code_quality/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/code/code_quality timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/code_quality workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/code_quality workflow-test echo "Run transforms/code/code_quality completed" else - echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/code/code_quality timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/code_quality workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/code_quality workflow-test echo "Run transforms/code/code_quality completed" else - echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml index 6cc4727aa..b1f4f1b1b 100644 --- a/.github/workflows/test-code-header_cleanser-kfp.yml +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/header_cleanser/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/header_cleanser/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/code/header_cleanser timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/header_cleanser/Makefile" -a -e "transforms/code/header_cleanser/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/header_cleanser workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/header_cleanser workflow-test echo "Run transforms/code/header_cleanser completed" else - echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/code/header_cleanser timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/header_cleanser/Makefile" -a -e "transforms/code/header_cleanser/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/header_cleanser workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/header_cleanser workflow-test echo "Run transforms/code/header_cleanser completed" else - echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml index 94d662d1d..0e953a09a 100644 --- a/.github/workflows/test-code-license_select-kfp.yml +++ b/.github/workflows/test-code-license_select-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/license_select/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/license_select/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/code/license_select timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/license_select/Makefile" -a -e "transforms/code/license_select/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/license_select workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/license_select workflow-test echo "Run transforms/code/license_select completed" else - echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/code/license_select timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/license_select/Makefile" -a -e "transforms/code/license_select/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/license_select workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/license_select workflow-test echo "Run transforms/code/license_select completed" else - echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml index 2c9e3186c..df580decb 100644 --- a/.github/workflows/test-code-malware-kfp.yml +++ b/.github/workflows/test-code-malware-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/malware/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/malware/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/code/malware timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/malware/Makefile" -a -e "transforms/code/malware/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/malware workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/malware workflow-test echo "Run transforms/code/malware completed" else - echo "Skipping transforms/code/malware kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/malware kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/code/malware timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/malware/Makefile" -a -e "transforms/code/malware/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/malware workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/malware workflow-test echo "Run transforms/code/malware completed" else - echo "Skipping transforms/code/malware kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/malware kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml index c23e0f1ff..b7ee58f2a 100644 --- a/.github/workflows/test-code-proglang_select-kfp.yml +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/proglang_select/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/proglang_select/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/code/proglang_select timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/proglang_select/Makefile" -a -e "transforms/code/proglang_select/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/proglang_select workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/proglang_select workflow-test echo "Run transforms/code/proglang_select completed" else - echo "Skipping transforms/code/proglang_select kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/proglang_select kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/code/proglang_select timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/proglang_select/Makefile" -a -e "transforms/code/proglang_select/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/proglang_select workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/proglang_select workflow-test echo "Run transforms/code/proglang_select completed" else - echo "Skipping transforms/code/proglang_select kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/proglang_select kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml index 57b39f313..3f11db5bb 100644 --- a/.github/workflows/test-code-repo_level_ordering-kfp.yml +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/repo_level_ordering/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/code/repo_level_ordering/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/code/repo_level_ordering timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/repo_level_ordering/Makefile" -a -e "transforms/code/repo_level_ordering/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/repo_level_ordering workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/repo_level_ordering workflow-test echo "Run transforms/code/repo_level_ordering completed" else - echo "Skipping transforms/code/repo_level_ordering kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/repo_level_ordering kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/code/repo_level_ordering timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/code/repo_level_ordering/Makefile" -a -e "transforms/code/repo_level_ordering/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/code/repo_level_ordering workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/repo_level_ordering workflow-test echo "Run transforms/code/repo_level_ordering completed" else - echo "Skipping transforms/code/repo_level_ordering kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/code/repo_level_ordering kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template index 1003ba643..29bba2634 100644 --- a/.github/workflows/test-kfp-transform.template +++ b/.github/workflows/test-kfp-transform.template @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "@TARGET_TRANSFORM_DIR@/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "@TARGET_TRANSFORM_DIR@/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for @TARGET_TRANSFORM_DIR@ timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" -a -e "@TARGET_TRANSFORM_DIR@/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C @TARGET_TRANSFORM_DIR@ workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C @TARGET_TRANSFORM_DIR@ workflow-test echo "Run @TARGET_TRANSFORM_DIR@ completed" else - echo "Skipping @TARGET_TRANSFORM_DIR@ kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping @TARGET_TRANSFORM_DIR@ kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for @TARGET_TRANSFORM_DIR@ timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" -a -e "@TARGET_TRANSFORM_DIR@/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C @TARGET_TRANSFORM_DIR@ workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C @TARGET_TRANSFORM_DIR@ workflow-test echo "Run @TARGET_TRANSFORM_DIR@ completed" else - echo "Skipping @TARGET_TRANSFORM_DIR@ kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping @TARGET_TRANSFORM_DIR@ kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml index 1c5237b20..35811aecb 100644 --- a/.github/workflows/test-language-doc_quality-kfp.yml +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/language/doc_quality/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/language/doc_quality/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/language/doc_quality timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/language/doc_quality/Makefile" -a -e "transforms/language/doc_quality/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/language/doc_quality workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/doc_quality workflow-test echo "Run transforms/language/doc_quality completed" else - echo "Skipping transforms/language/doc_quality kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/language/doc_quality kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/language/doc_quality timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/language/doc_quality/Makefile" -a -e "transforms/language/doc_quality/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/language/doc_quality workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/doc_quality workflow-test echo "Run transforms/language/doc_quality completed" else - echo "Skipping transforms/language/doc_quality kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/language/doc_quality kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml index c6eb179b8..a6cd597b2 100644 --- a/.github/workflows/test-language-lang_id-kfp.yml +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/language/lang_id/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/language/lang_id/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/language/lang_id timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/language/lang_id/Makefile" -a -e "transforms/language/lang_id/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/language/lang_id workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/lang_id workflow-test echo "Run transforms/language/lang_id completed" else - echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/language/lang_id timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/language/lang_id/Makefile" -a -e "transforms/language/lang_id/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/language/lang_id workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/lang_id workflow-test echo "Run transforms/language/lang_id completed" else - echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml index 8e238dfcf..3f74c5861 100644 --- a/.github/workflows/test-language-text_encoder-kfp.yml +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/language/text_encoder/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/language/text_encoder/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/language/text_encoder timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/language/text_encoder/Makefile" -a -e "transforms/language/text_encoder/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/language/text_encoder workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/text_encoder workflow-test echo "Run transforms/language/text_encoder completed" else - echo "Skipping transforms/language/text_encoder kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/language/text_encoder kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/language/text_encoder timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/language/text_encoder/Makefile" -a -e "transforms/language/text_encoder/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/language/text_encoder workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/text_encoder workflow-test echo "Run transforms/language/text_encoder completed" else - echo "Skipping transforms/language/text_encoder kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/language/text_encoder kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml index 8ed1df919..f40848f62 100644 --- a/.github/workflows/test-universal-doc_id-kfp.yml +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/doc_id/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/doc_id/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/universal/doc_id timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/doc_id/Makefile" -a -e "transforms/universal/doc_id/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/doc_id workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/doc_id workflow-test echo "Run transforms/universal/doc_id completed" else - echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/universal/doc_id timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/doc_id/Makefile" -a -e "transforms/universal/doc_id/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/doc_id workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/doc_id workflow-test echo "Run transforms/universal/doc_id completed" else - echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml index 93408a260..b3edeec7d 100644 --- a/.github/workflows/test-universal-ededup-kfp.yml +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/ededup/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/ededup/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/universal/ededup timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/ededup/Makefile" -a -e "transforms/universal/ededup/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/ededup workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/ededup workflow-test echo "Run transforms/universal/ededup completed" else - echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/universal/ededup timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/ededup/Makefile" -a -e "transforms/universal/ededup/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/ededup workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/ededup workflow-test echo "Run transforms/universal/ededup completed" else - echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml index 27dfc5493..8d18552c3 100644 --- a/.github/workflows/test-universal-fdedup-kfp.yml +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/fdedup/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/fdedup/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/universal/fdedup timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/fdedup/Makefile" -a -e "transforms/universal/fdedup/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/fdedup workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/fdedup workflow-test echo "Run transforms/universal/fdedup completed" else - echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/universal/fdedup timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/fdedup/Makefile" -a -e "transforms/universal/fdedup/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/fdedup workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/fdedup workflow-test echo "Run transforms/universal/fdedup completed" else - echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml index bd2f57229..fde070bad 100644 --- a/.github/workflows/test-universal-filter-kfp.yml +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/filter/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/filter/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/universal/filter timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/filter/Makefile" -a -e "transforms/universal/filter/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/filter workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/filter workflow-test echo "Run transforms/universal/filter completed" else - echo "Skipping transforms/universal/filter kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/filter kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/universal/filter timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/filter/Makefile" -a -e "transforms/universal/filter/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/filter workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/filter workflow-test echo "Run transforms/universal/filter completed" else - echo "Skipping transforms/universal/filter kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/filter kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml index 01b14e51b..0aa16a984 100644 --- a/.github/workflows/test-universal-noop-kfp.yml +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/noop/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/noop/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/universal/noop timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/noop/Makefile" -a -e "transforms/universal/noop/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/noop workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/noop workflow-test echo "Run transforms/universal/noop completed" else - echo "Skipping transforms/universal/noop kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/noop kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/universal/noop timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/noop/Makefile" -a -e "transforms/universal/noop/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/noop workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/noop workflow-test echo "Run transforms/universal/noop completed" else - echo "Skipping transforms/universal/noop kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/noop kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml index e30f7bafa..7f8451f52 100644 --- a/.github/workflows/test-universal-profiler-kfp.yml +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/profiler/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/profiler/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/universal/profiler timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/profiler/Makefile" -a -e "transforms/universal/profiler/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/profiler workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/profiler workflow-test echo "Run transforms/universal/profiler completed" else - echo "Skipping transforms/universal/profiler kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/profiler kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/universal/profiler timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/profiler/Makefile" -a -e "transforms/universal/profiler/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/profiler workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/profiler workflow-test echo "Run transforms/universal/profiler completed" else - echo "Skipping transforms/universal/profiler kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/profiler kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml index 630de3c05..7cee28ff5 100644 --- a/.github/workflows/test-universal-resize-kfp.yml +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/resize/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/resize/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/universal/resize timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/resize/Makefile" -a -e "transforms/universal/resize/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/resize workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/resize workflow-test echo "Run transforms/universal/resize completed" else - echo "Skipping transforms/universal/resize kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/resize kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/universal/resize timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/resize/Makefile" -a -e "transforms/universal/resize/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/resize workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/resize workflow-test echo "Run transforms/universal/resize completed" else - echo "Skipping transforms/universal/resize kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/resize kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml index ff13a444c..8d98b3be3 100644 --- a/.github/workflows/test-universal-tokenization-kfp.yml +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -13,7 +13,7 @@ on: - "*" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/tokenization/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -27,7 +27,7 @@ on: - "releases/**" paths: - ".make.*" - - "transforms/.make.workflow" + - "transforms/.make.workflows" - "transforms/universal/tokenization/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -56,10 +56,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v1) and run a workflow + - name: Test V1 KFP workflow for transforms/universal/tokenization timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/tokenization/Makefile" -a -e "transforms/universal/tokenization/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -76,13 +76,13 @@ jobs: chmod +x /tmp/mc export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/tokenization workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/tokenization workflow-test echo "Run transforms/universal/tokenization completed" else - echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray/Makefile" fi test-kfp-v2: @@ -99,10 +99,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test KFP libs (shared and v2) and run a workflow + - name: Test V2 KFP workflow for transforms/universal/tokenization timeout-minutes: 120 run: | - if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + if [ -e "transforms/universal/tokenization/Makefile" -a -e "transforms/universal/tokenization/kfp_ray/Makefile" ]; then export REPOROOT=$PWD export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup source $K8S_SETUP_SCRIPTS/requirements.env @@ -120,11 +120,11 @@ jobs: export DEPLOY_KUBEFLOW=1 export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test + #make -C kfp/kfp_support_lib test make -C transforms/universal/tokenization workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/tokenization workflow-test echo "Run transforms/universal/tokenization completed" else - echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray" + echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray/Makefile" fi diff --git a/transforms/.make.workflows b/transforms/.make.workflows index adbf721e6..9b7259b60 100644 --- a/transforms/.make.workflows +++ b/transforms/.make.workflows @@ -9,7 +9,7 @@ define set_env_var endef -KFP_ENDPOINT ?= "http://localhost:8080/" +KFP_ENDPOINT?="http://localhost:8080/" .PHONY: set-versions set-versions: diff --git a/transforms/code/license_select/kfp_ray/Makefile b/transforms/code/license_select/kfp_ray/Makefile.disable-cicd similarity index 100% rename from transforms/code/license_select/kfp_ray/Makefile rename to transforms/code/license_select/kfp_ray/Makefile.disable-cicd diff --git a/transforms/code/repo_level_ordering/kfp_ray/Makefile b/transforms/code/repo_level_ordering/kfp_ray/Makefile.disable-cicd similarity index 100% rename from transforms/code/repo_level_ordering/kfp_ray/Makefile rename to transforms/code/repo_level_ordering/kfp_ray/Makefile.disable-cicd diff --git a/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py b/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py index a36cc9c0c..7ec37bd9d 100644 --- a/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py +++ b/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py @@ -122,7 +122,14 @@ def repo_level_order( # Ray cluster ray_name: str = "repo_level_order-kfp-ray", ray_head_options: dict = {"cpu": 1, "memory": 4, "image": task_image}, - ray_worker_options: dict = {"replicas": 2, "max_replicas": 2, "min_replicas": 2, "cpu": 2, "memory": 4, "image": task_image}, + ray_worker_options: dict = { + "replicas": 2, + "max_replicas": 2, + "min_replicas": 2, + "cpu": 2, + "memory": 4, + "image": task_image, + }, server_url: str = "http://kuberay-apiserver-service.kuberay.svc.cluster.local:8888", # data access data_s3_config: str = "{'input_folder': 'test/repo_level_ordering/input', 'output_folder': 'test/repo_level_ordering/output'}", @@ -130,9 +137,9 @@ def repo_level_order( data_max_files: int = -1, data_num_samples: int = -1, # orchestrator - runtime_actor_options: dict = {'num_cpus': 0.8}, + runtime_actor_options: dict = {"num_cpus": 0.8}, runtime_pipeline_id: str = "pipeline_id", - runtime_code_location: dict = {'github': 'github', 'commit_hash': '12345', 'path': 'path'}, + runtime_code_location: dict = {"github": "github", "commit_hash": "12345", "path": "path"}, # repo_level_order parameters repo_lvl_stage_one_only: bool = False, repo_lvl_grouping_column: str = "repo_name", @@ -193,7 +200,9 @@ def repo_level_order( :return: None """ # create clean_up task - clean_up_task = cleanup_ray_op(ray_name=ray_name, run_id=run_id, server_url=server_url, additional_params=additional_params) + clean_up_task = cleanup_ray_op( + ray_name=ray_name, run_id=run_id, server_url=server_url, additional_params=additional_params + ) ComponentUtils.add_settings_to_component(clean_up_task, ONE_HOUR_SEC * 2) # pipeline definition with dsl.ExitHandler(clean_up_task): diff --git a/transforms/language/text_encoder/kfp_ray/Makefile b/transforms/language/text_encoder/kfp_ray/Makefile.disable-cicd similarity index 100% rename from transforms/language/text_encoder/kfp_ray/Makefile rename to transforms/language/text_encoder/kfp_ray/Makefile.disable-cicd