From 4c9c9b3ff682947abf0c34779c840f41e924a195 Mon Sep 17 00:00:00 2001 From: Yangkuan Liu Date: Sat, 30 Nov 2024 20:16:34 +0800 Subject: [PATCH] support juicefs-csi-node run as ads --- cmd/controller.go | 19 +++++++++++++++---- deploy/k8s.yaml | 8 ++++++++ deploy/k8s_before_v1_18.yaml | 7 +++++++ deploy/kubernetes/base/resources.yaml | 3 +++ deploy/webhook-with-certmanager.yaml | 7 +++++++ deploy/webhook.yaml | 7 +++++++ go.mod | 4 +++- go.sum | 4 ++++ pkg/k8sclient/client.go | 20 +++++++++++++++++++- scripts/juicefs-csi-webhook-install.sh | 14 ++++++++++++++ 10 files changed, 87 insertions(+), 6 deletions(-) diff --git a/cmd/controller.go b/cmd/controller.go index 829d3b4340..2cccad6410 100644 --- a/cmd/controller.go +++ b/cmd/controller.go @@ -114,12 +114,23 @@ func parseControllerConfig() { CSINodeDsName = name } ds, err := k8sclient.GetDaemonSet(context.TODO(), CSINodeDsName, config.Namespace) + ads, err2 := k8sclient.GetADaemonSet(context.TODO(), CSINodeDsName, config.Namespace) if err != nil { - log.Error(err, "Can't get DaemonSet", "ds", CSINodeDsName) - os.Exit(1) + log.Error(err, "Can't get DaemonSet, try get ads", "ds", CSINodeDsName) } - config.CSIPod = corev1.Pod{ - Spec: ds.Spec.Template.Spec, + + if ds != nil { + config.CSIPod = corev1.Pod{ + Spec: ds.Spec.Template.Spec, + } + } else if ads != nil { + config.CSIPod = corev1.Pod{ + Spec: ads.Spec.Template.Spec, + } + log.Info("get ads successful", "ds", CSINodeDsName) + } else { + log.Error(err2, "Can't get ds or ads", "ds", CSINodeDsName) + os.Exit(1) } } } diff --git a/deploy/k8s.yaml b/deploy/k8s.yaml index ece39242d6..f6293f106e 100644 --- a/deploy/k8s.yaml +++ b/deploy/k8s.yaml @@ -339,6 +339,13 @@ rules: verbs: - get - list +- apiGroups: + - apps.kruise.io + resources: + - daemonsets + verbs: + - get + - list - apiGroups: - coordination.k8s.io resources: @@ -650,6 +657,7 @@ spec: name: juicefs-config volumeClaimTemplates: [] --- +#apiVersion: apps.kruise.io/v1alpha1 apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/deploy/k8s_before_v1_18.yaml b/deploy/k8s_before_v1_18.yaml index 0de7946198..ab2ef6d363 100644 --- a/deploy/k8s_before_v1_18.yaml +++ b/deploy/k8s_before_v1_18.yaml @@ -339,6 +339,13 @@ rules: verbs: - get - list +- apiGroups: + - apps.kruise.io + resources: + - daemonsets + verbs: + - get + - list - apiGroups: - coordination.k8s.io resources: diff --git a/deploy/kubernetes/base/resources.yaml b/deploy/kubernetes/base/resources.yaml index 4f6948d943..680f078458 100644 --- a/deploy/kubernetes/base/resources.yaml +++ b/deploy/kubernetes/base/resources.yaml @@ -44,6 +44,9 @@ rules: - apiGroups: ["apps"] resources: ["daemonsets"] verbs: ["get", "list"] + - apiGroups: [ "apps.kruise.io" ] + resources: [ "daemonsets" ] + verbs: [ "get", "list" ] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] diff --git a/deploy/webhook-with-certmanager.yaml b/deploy/webhook-with-certmanager.yaml index 9cef68799e..01153417b1 100644 --- a/deploy/webhook-with-certmanager.yaml +++ b/deploy/webhook-with-certmanager.yaml @@ -247,6 +247,13 @@ rules: verbs: - get - list +- apiGroups: + - apps.kruise.io + resources: + - daemonsets + verbs: + - get + - list - apiGroups: - coordination.k8s.io resources: diff --git a/deploy/webhook.yaml b/deploy/webhook.yaml index d10d71b0f1..ce86dfe736 100644 --- a/deploy/webhook.yaml +++ b/deploy/webhook.yaml @@ -247,6 +247,13 @@ rules: verbs: - get - list +- apiGroups: + - apps.kruise.io + resources: + - daemonsets + verbs: + - get + - list - apiGroups: - coordination.k8s.io resources: diff --git a/go.mod b/go.mod index 0db99a0123..e9b7364b24 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/kubernetes-csi/csi-test/v5 v5.3.1 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 + github.com/openkruise/kruise-api v1.4.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 github.com/smartystreets/goconvey v1.6.4 @@ -26,6 +27,7 @@ require ( k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.19.0 sigs.k8s.io/sig-storage-lib-external-provisioner/v10 v10.0.1 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -101,9 +103,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + sigs.k8s.io/gateway-api v0.6.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) go 1.22.0 diff --git a/go.sum b/go.sum index 228c3e96bf..64da334444 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,8 @@ github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/openkruise/kruise-api v1.4.0 h1:MDDXQIYvaCh0ioIJSRniF4kCKby9JI3/ec6pZHHw/Ao= +github.com/openkruise/kruise-api v1.4.0/go.mod h1:HyRlDV0MfW5Zm+3g36bx7u4CcWHcKBxL8g/c/2bjcd4= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -294,6 +296,8 @@ nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYm rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/gateway-api v0.6.2 h1:583XHiX2M2bKEA0SAdkoxL1nY73W1+/M+IAm8LJvbEA= +sigs.k8s.io/gateway-api v0.6.2/go.mod h1:EYJT+jlPWTeNskjV0JTki/03WX1cyAnBhwBJfYHpV/0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/sig-storage-lib-external-provisioner/v10 v10.0.1 h1:uhd7RobUnVmfkRb3gkYQh4tBOiBEBvdwW/nAthG95Rc= diff --git a/pkg/k8sclient/client.go b/pkg/k8sclient/client.go index 98763c691c..254c3f3e11 100644 --- a/pkg/k8sclient/client.go +++ b/pkg/k8sclient/client.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "fmt" + "github.com/openkruise/kruise-api/apps/v1alpha1" "io" "net/url" "os" @@ -27,6 +28,7 @@ import ( "strings" "time" + kruiseclientset "github.com/openkruise/kruise-api/client/clientset/versioned" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" appsv1 "k8s.io/api/apps/v1" @@ -78,6 +80,7 @@ type PatchDelValue struct { type K8sClient struct { enableAPIServerListCache bool + kruiseClient kruiseclientset.Interface kubernetes.Interface } @@ -122,7 +125,11 @@ func newClient(config *rest.Config) (*K8sClient, error) { if os.Getenv("ENABLE_APISERVER_LIST_CACHE") == "true" { enableAPIServerListCache = true } - return &K8sClient{enableAPIServerListCache, client}, nil + kruiseClient, err2 := kruiseclientset.NewForConfig(config) + if err2 != nil { + return nil, err2 + } + return &K8sClient{enableAPIServerListCache, kruiseClient, client}, nil } func (k *K8sClient) CreatePod(ctx context.Context, pod *corev1.Pod) (*corev1.Pod, error) { @@ -457,6 +464,17 @@ func (k *K8sClient) GetStorageClass(ctx context.Context, scName string) (*storag return mntPod, nil } +func (k *K8sClient) GetADaemonSet(ctx context.Context, dsName, namespace string) (*v1alpha1.DaemonSet, error) { + log := util.GenLog(ctx, clientLog, "") + log.V(1).Info("Get ads", "name", dsName) + ads, err := k.kruiseClient.AppsV1alpha1().DaemonSets(namespace).Get(ctx, dsName, metav1.GetOptions{}) + if err != nil { + log.Info("Can't get ADaemonSet", "ds", dsName) + return nil, err + } + return ads, nil +} + func (k *K8sClient) GetDaemonSet(ctx context.Context, dsName, namespace string) (*appsv1.DaemonSet, error) { log := util.GenLog(ctx, clientLog, "") log.V(1).Info("Get ds", "name", dsName) diff --git a/scripts/juicefs-csi-webhook-install.sh b/scripts/juicefs-csi-webhook-install.sh index a276803856..8dcbff0a1e 100755 --- a/scripts/juicefs-csi-webhook-install.sh +++ b/scripts/juicefs-csi-webhook-install.sh @@ -318,6 +318,13 @@ rules: verbs: - get - list +- apiGroups: + - apps.kruise.io + resources: + - daemonsets + verbs: + - get + - list - apiGroups: - coordination.k8s.io resources: @@ -1073,6 +1080,13 @@ rules: verbs: - get - list +- apiGroups: + - apps.kruise.io + resources: + - daemonsets + verbs: + - get + - list - apiGroups: - coordination.k8s.io resources: