arm (secretflow#283)

imwangyt · Apr 24, 2024 · 245a38a · 245a38a
1 parent 88bb2ec
commit 245a38a
Show file tree

Hide file tree

Showing 13 changed files with 273 additions and 96 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -20,64 +20,123 @@ parameters:
   GHA_Meta:
     type: string
     default: ""
+  DEPS_Tag:
+    type: string
+    default: ""
 
-# Define a job to be invoked later in a workflow.
-# See: https://circleci.com/docs/configuration-reference/#jobs
-jobs:
-  unit-test:
+executors:
+  linux_x64_executor: # declares a reusable executor
     docker:
       - image: secretflow/kuscia-ci:0.4
     resource_class: 2xlarge
     shell: /bin/bash --login -eo pipefail
+  linux_aarch64_executor:
+    docker:
+      - image: secretflow/ubuntu-base-ci:latest
+    resource_class: arm.2xlarge
+    shell: /bin/bash --login -eo pipefail
+
+commands:
+  build_kuscia:
     steps:
-      - checkout
       - setup_remote_docker
       - run:
-          name: Test Kuscia
-          command: make test
+          name: "install go"
+          command: |
+            ARCH=$(uname -m)
+            case "$ARCH" in
+            x86_64) ARCH="amd64";;
+            aarch64) ARCH="arm64";;
+            *) echo "Unsupported architecture"; exit 1;;
+            esac
+
+            GOLANG_DIR="/usr/local"
+            GOLANG_VERSION="1.22.2"
+            GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+
+            if ! command -v go &> /dev/null; then
+            wget "$GOLANG_URL"
+            tar -C "$GOLANG_DIR" -xzf "go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+            echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a ~/.bashrc
+            echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV
+            source ~/.bashrc
+            fi
+      - run:
+          name: "make build"
+          command: |
+            git config --global --add safe.directory ./
+            make build
+      - store_artifacts:
+          path: build
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/configuration-reference/#jobs
+jobs:
+  linux_build_kuscia:
+    parameters:
+      executor:
+        type: string
+    executor: <<parameters.executor>>
+    steps:
+      - checkout
+      - build_kuscia
   image_publish:
     docker:
       - image: secretflow/kuscia-ci:0.4
     shell: /bin/bash --login -eo pipefail
     steps:
+      - attach_workspace:
+          at: build
       - checkout
       - setup_remote_docker
       - run:
-          name: Login to kuscia docker registry
+          name: Build image and publish
           command: |
+            CIRCLETAG=$(echo ${CIRCLE_TAG} | sed 's/v//')
+            IMAGE = "secretflow/kuscia"
+            ALIYUN_IMAGE = "secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia"
+            
             # login kuscia dockerhub registry
             docker login -u secretflow -p ${DOCKER_DEPLOY_TOKEN}
+            
+            docker buildx create --name kuscia --platform linux/arm64,linux/amd64 --use
+            
+            docker buildx build -t ${IMAGE}:${CIRCLETAG} --platform linux/arm64,linux/amd64 -f ./build/dockerfile/kuscia-anolis.Dockerfile . --push
+            docker buildx build -t ${IMAGE}:latest --platform linux/arm64,linux/amd64 -f ./build/dockerfile/kuscia-anolis.Dockerfile . --push
+            
             # login kuscia aliyun registry
             docker login -u ${ALIYUN_DOCKER_USERNAME} -p ${ALIYUN_DOCKER_PASSWORD}  secretflow-registry.cn-hangzhou.cr.aliyuncs.com
-      - run:
-          name: Build image and publish
-          command: |
-            make image
-            CIRCLETAG=$(echo ${CIRCLE_TAG} | sed 's/v//')
-            docker tag $(docker images --format "{{.Repository}}:{{.Tag}}" | sed -n '1p') secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia:latest
-            docker tag $(docker images --format "{{.Repository}}:{{.Tag}}" | sed -n '1p') secretflow/kuscia:latest
-            docker tag $(docker images --format "{{.Repository}}:{{.Tag}}" | sed -n '1p') secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia:${CIRCLETAG}
-            docker push secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia:${CIRCLETAG}
-            docker tag $(docker images --format "{{.Repository}}:{{.Tag}}" | sed -n '1p') secretflow/kuscia:${CIRCLETAG}
-            docker push secretflow/kuscia:${CIRCLETAG}
-            docker push secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia:latest
-            docker push secretflow/kuscia:latest
+            
+            docker buildx build -t ${ALIYUN_IMAGE}:latest --platform linux/arm64,linux/amd64 -f ./build/dockerfile/kuscia-anolis.Dockerfile . --push
+            docker buildx build -t ${ALIYUN_IMAGE}:${CIRCLETAG} --platform linux/arm64,linux/amd64 -f ./build/dockerfile/kuscia-anolis.Dockerfile . --push
+
 
 # Orchestrate jobs using workflows
 # See: https://circleci.com/docs/configuration-reference/#workflows
 workflows:
   build-workflow:
     when:
-      not:
-        equal: [ "publish_pypi", << pipeline.parameters.GHA_Meta >> ]
+      or:
+        - equal: [ "publish_kuscia_deps", << pipeline.parameters.GHA_Meta >> ]
+        - equal: [ "publish_pypi", << pipeline.parameters.GHA_Meta >> ]
     jobs:
-      - unit-test
+      - linux_build_kuscia:
+          matrix:
+            parameters:
+              executor: [ "linux_x64_executor", "linux_aarch64_executor" ]
       - image_publish:
+          requires:
+            - linux_build_kuscia
           filters:
             branches:
               ignore: /.*/
             tags:
               only: /^v.*/
+  build-kuscia-deps-workflow:
+    when:
+      equal: [ "publish_kuscia_deps", << pipeline.parameters.GHA_Meta >> ]
+    jobs:
+      - continuation/continue:
+          configuration_path: .circleci/deps-config.yml
   publish-workflow:
     when:
       equal: [ "publish_pypi", << pipeline.parameters.GHA_Meta >> ]

diff --git a/.circleci/deps-config.yml b/.circleci/deps-config.yml
@@ -0,0 +1,98 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/configuration-reference
+version: 2.1
+
+parameters:
+  DEPS_Tag:
+    type: string
+    default: ""
+
+executors:
+  linux_x64_executor: #  declares a reusable executor
+    docker:
+      - image: secretflow/kuscia-ci:0.4
+    resource_class: 2xlarge
+    shell: /bin/bash --login -eo pipefail
+  linux_aarch64_executor:
+    docker:
+      - image: secretflow/ubuntu-base-ci:latest
+    resource_class: arm.2xlarge
+    shell: /bin/bash --login -eo pipefail
+
+commands:
+  build_kuscia_deps:
+    steps:
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: "install go"
+          command: |
+            ARCH=$(uname -m)
+            case "$ARCH" in
+            x86_64) ARCH="amd64";;
+            aarch64) ARCH="arm64";;
+            *) echo "Unsupported architecture"; exit 1;;
+            esac
+
+            GOLANG_DIR="/opt"
+            GOLANG_VERSION="1.22.2"
+            GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+
+            if ! command -v go &> /dev/null; then
+            wget "$GOLANG_URL"
+            
+            tar -C "$GOLANG_DIR" -xzf "go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+            echo 'export PATH=$PATH:/opt/go/bin' | tee -a ~/.bashrc
+            echo 'export PATH=$PATH:/opt/go/bin' >> $BASH_ENV
+            source ~/.bashrc
+            fi
+      - run:
+          name: "make build"
+          command: |
+            git config --global --add safe.directory ./
+            make deps-build
+      - store_artifacts:
+          path: build
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/configuration-reference/#jobs
+jobs:
+  linux_build_kuscia_deps:
+    parameters:
+      executor:
+        type: string
+    executor: <<parameters.executor>>
+    steps:
+      - checkout
+      - build_kuscia_deps
+  image_publish:
+    docker:
+      - image: secretflow/kuscia-ci:0.4
+    shell: /bin/bash --login -eo pipefail
+    steps:
+      - attach_workspace:
+          at: build
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Build image and publish
+          command: |
+            DEPS_IMAGE = secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:<< pipeline.parameters.DEPS_Tag >>
+             # login kuscia aliyun registry
+            docker login -u ${ALIYUN_DOCKER_USERNAME} -p ${ALIYUN_DOCKER_PASSWORD}  secretflow-registry.cn-hangzhou.cr.aliyuncs.com
+            
+            docker buildx create --name kuscia_deps --platform linux/arm64,linux/amd64 --use
+            
+            docker buildx build -t ${DEPS_IMAGE}:${CIRCLETAG} --platform linux/arm64,linux/amd64 -f ./build/dockerfile/base/kuscia-deps.Dockerfile . --push
+           
+            
+
+workflows:
+  build_deps_workflow:
+    jobs:
+      - linux_build_kuscia_deps:
+          matrix:
+            parameters:
+              executor: [ "linux_x64_executor", "linux_aarch64_executor" ]
+      - image_publish:
+          requires:
+            - linux_build_kuscia_deps
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 /.idea/
 /build/apps/
 /build/k3s/
+/build/linux/
 /etc/certs/
 **.log
 .vscode

diff --git a/Makefile b/Makefile
@@ -2,6 +2,13 @@
 # Image URL to use all building image targets
 DATETIME = $(shell date +"%Y%m%d%H%M%S")
 KUSCIA_VERSION_TAG = $(shell git describe --tags --always)
+# Get current architecture information
+UNAME_M_OUTPUT := $(shell uname -m)
+
+# To configure the ARCH variable to either arm64 or amd64 or UNAME_M_OUTPUT
+ARCH := $(if $(filter aarch64 arm64,$(UNAME_M_OUTPUT)),arm64,$(if $(filter amd64 x86_64,$(UNAME_M_OUTPUT)),amd64,$(UNAME_M_OUTPUT)))
+
+
 TAG = ${KUSCIA_VERSION_TAG}-${DATETIME}
 IMG := secretflow/kuscia:${TAG}
 
@@ -95,25 +102,32 @@ clean: # clean build and test product.
 	-rm -rf ./build/apps
 	-rm -rf ./build/framework
 	-rm -rf ./tmp-crd-code
+	-rm -rf ./build/linux
 
 ##@ Build
 
 .PHONY: build
 build: verify_error_code fmt vet ## Build kuscia binary.
 	bash hack/build.sh -t kuscia
-
+	mkdir -p build/linux/${ARCH}
+	mv build/apps build/linux/${ARCH}
 .PHONY: docs
 docs: gen_error_code_doc ## Build docs.
 	cd docs && pip install -r requirements.txt && make html
 
-.PHONY: deps-image
-deps-image:
+.PHONY: deps-build
+deps-build:
 	bash hack/k3s/build.sh
+	mkdir -p build/linux/${ARCH}
+	mv build/k3s build/linux/${ARCH}
+
+.PHONY: deps-image
+deps-image: deps-build
 	docker build -t ${DEPS_IMAGE} -f ./build/dockerfile/base/kuscia-deps.Dockerfile .
 
 .PHONY: image
 image: export GOOS=linux
-image: export GOARCH=amd64
+image: export GOARCH=${ARCH}
 image: build ## Build docker image with the manager.
 	docker build -t ${IMG} --build-arg KUSCIA_ENVOY_IMAGE=${ENVOY_IMAGE} --build-arg DEPS_IMAGE=${DEPS_IMAGE} -f ./build/dockerfile/kuscia-anolis.Dockerfile .
 

diff --git a/build/dockerfile/base/kuscia-deps.Dockerfile b/build/dockerfile/base/kuscia-deps.Dockerfile
@@ -1,10 +1,10 @@
-ARG ARCH=amd64
 ARG K3S_VER=v1.26.11-k3s2
-ARG K3S_IMAGE=rancher/k3s:${K3S_VER}-${ARCH}
+ARG K3S_IMAGE=rancher/k3s:${K3S_VER}
 FROM ${K3S_IMAGE} as k3s-image
 
 FROM openanolis/anolisos:8.8
-
+ARG TARGETPLATFORM
+ARG TARGETARCH
 RUN yum install -y git glibc-static wget gcc make && \
     yum clean all
 
@@ -17,8 +17,7 @@ WORKDIR /tmp
 COPY --from=k3s-image /bin/k3s /bin/containerd /bin/containerd-shim-runc-v2 /bin/runc /bin/cni /image/home/kuscia/bin/
 COPY --from=k3s-image /bin/aux /image/bin/aux
 
-COPY build/k3s/bin/k3s /image/home/kuscia/bin/
-
-RUN wget https://github.com/krallin/tini/releases/download/v0.19.0/tini -O /image/home/kuscia/bin/tini && \
-    chmod +x /image/home/kuscia/bin/tini
+COPY build/${TARGETPLATFORM}/k3s /image/home/kuscia/bin/
 
+RUN wget "https://github.com/krallin/tini/releases/download/v0.19.0/tini-${TARGETARCH}" -O /image/home/kuscia/bin/tini; \
+    chmod +x /image/home/kuscia/bin/tini;
diff --git a/build/dockerfile/kuscia-anolis.Dockerfile b/build/dockerfile/kuscia-anolis.Dockerfile
@@ -1,15 +1,18 @@
+ARG ARCH=amd64
 ARG DEPS_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0"
-ARG KUSCIA_ENVOY_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240402"
+ARG KUSCIA_ENVOY_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.5.0b0"
 ARG PROM_NODE_EXPORTER="prom/node-exporter:v1.7.0"
 
 FROM ${DEPS_IMAGE} as deps
+
 FROM ${PROM_NODE_EXPORTER} as node_exporter
 FROM ${KUSCIA_ENVOY_IMAGE} as kuscia_envoy
 
 FROM openanolis/anolisos:8.8
 
 ENV TZ=Asia/Shanghai
-
+ARG TARGETPLATFORM
+ARG TARGETARCH
 ARG ROOT_DIR="/home/kuscia"
 RUN yum install -y openssl net-tools which jq logrotate && \
     yum clean all && \
@@ -33,8 +36,8 @@ RUN pushd ${ROOT_DIR}/bin && \
     ln -s cni portmap && \
     popd
 
-COPY build/apps/kuscia/kuscia ${ROOT_DIR}/bin
-COPY build/pause/pause.tar ${ROOT_DIR}/pause
+COPY build/${TARGETPLATFORM}/apps/kuscia/kuscia ${ROOT_DIR}/bin
+COPY build/pause/pause-${TARGETARCH}.tar ${ROOT_DIR}/pause/pause.tar
 COPY crds/v1alpha1 ${ROOT_DIR}/crds/v1alpha1
 COPY etc ${ROOT_DIR}/etc
 COPY testdata ${ROOT_DIR}/var/storage/data

diff --git a/build/pause/pause.tar → build/pause/pause-amd64.tar b/build/pause/pause.tar → build/pause/pause-amd64.tar
diff --git a/build/pause/pause-arm64.tar b/build/pause/pause-arm64.tar
diff --git a/scripts/deploy/deploy.sh b/scripts/deploy/deploy.sh
@@ -29,8 +29,7 @@ function log() {
 function arch_check() {
   local arch=$(uname -a)
   if [[ $arch == *"ARM"* ]] || [[ $arch == *"aarch64"* ]]; then
-    echo -e "${RED}ARM architecture is not supported by kuscia currently${NC}"
-    exit 1
+    echo "Warning: arm64 architecture. Continuing..."
   elif [[ $arch == *"x86_64"* ]]; then
     echo -e "${GREEN}x86_64 architecture. Continuing...${NC}"
   elif [[ $arch == *"amd64"* ]]; then

diff --git a/scripts/deploy/kuscia.sh b/scripts/deploy/kuscia.sh
@@ -56,8 +56,7 @@ function log() {
 function arch_check() {
   local arch=$(uname -a)
   if [[ $arch == *"ARM"* ]] || [[ $arch == *"aarch64"* ]]; then
-    echo -e "${RED}ARM architecture is not supported by kuscia currently${NC}"
-    exit 1
+    echo "Warning: arm64 architecture. Continuing..."
   elif [[ $arch == *"x86_64"* ]]; then
     echo -e "${GREEN}x86_64 architecture. Continuing...${NC}"
   elif [[ $arch == *"amd64"* ]]; then

diff --git a/scripts/deploy/start_standalone.sh b/scripts/deploy/start_standalone.sh
@@ -62,8 +62,7 @@ function pre_check() {
 function arch_check() {
   local arch=$(uname -a)
   if [[ $arch == *"ARM"* ]] || [[ $arch == *"aarch64"* ]]; then
-    echo -e "${RED}ARM architecture is not supported by kuscia currently${NC}"
-    exit 1
+    echo "Warning: arm64 architecture. Continuing..."
   elif [[ $arch == *"x86_64"* ]]; then
     echo -e "${GREEN}x86_64 architecture. Continuing...${NC}"
   elif [[ $arch == *"amd64"* ]]; then