From 2d483b01f991303621862f6880177abb673b0a09 Mon Sep 17 00:00:00 2001 From: Asher Liu Date: Tue, 26 Jul 2022 13:32:24 +0800 Subject: [PATCH] fix: cluster agent token expire in kubernetes >= 1.21 (#5280) --- cmd/cluster-agent/bootstrap.yaml | 2 + go.mod | 2 +- go.sum | 6 +- internal/tools/cluster-agent/config/config.go | 2 + .../tools/cluster-agent/pkg/client/client.go | 69 ++++++++++++++++--- .../cluster-agent/pkg/client/client_test.go | 65 +++++++++++++++++ 6 files changed, 132 insertions(+), 14 deletions(-) create mode 100644 internal/tools/cluster-agent/pkg/client/client_test.go diff --git a/cmd/cluster-agent/bootstrap.yaml b/cmd/cluster-agent/bootstrap.yaml index 1415ffd3d21..02942f155ff 100644 --- a/cmd/cluster-agent/bootstrap.yaml +++ b/cmd/cluster-agent/bootstrap.yaml @@ -1,5 +1,7 @@ cluster-agent: debug: ${DEBUG:false} + collectSource: "${COLLECT_SOURCE:secret}" + serviceAccountName: "${SERVICE_ACCOUNT_NAME:cluster-agent}" leaderElection: "${LEADER_ELECTION:true}" leaderElectionID: "${LEADER_ELECTION_ID:cluster-agent.erda.cloud}" leasesResourceLockType: "${LEASES_RESOURCE_LOCK_TYPE:leases}" diff --git a/go.mod b/go.mod index 6d68c2008da..fe23b130d97 100644 --- a/go.mod +++ b/go.mod @@ -150,7 +150,7 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/afero v1.6.0 github.com/spf13/cast v1.3.1 - github.com/spf13/cobra v1.4.0 + github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.7.1 github.com/t-tiger/gorm-bulk-insert v1.3.0 github.com/tealeg/xlsx v1.0.5 diff --git a/go.sum b/go.sum index e7a196711d7..e04f3d87f41 100644 --- a/go.sum +++ b/go.sum @@ -407,7 +407,7 @@ github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawk github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= @@ -1706,8 +1706,8 @@ github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= diff --git a/internal/tools/cluster-agent/config/config.go b/internal/tools/cluster-agent/config/config.go index ab73bf6d4e0..ba34c5806b1 100644 --- a/internal/tools/cluster-agent/config/config.go +++ b/internal/tools/cluster-agent/config/config.go @@ -17,6 +17,8 @@ package config type Config struct { Debug bool `default:"false" desc:"enable debug logging"` CollectClusterInfo bool `default:"true" desc:"enable collect cluster info"` + CollectSource string `default:"secret" desc:"collect source, secret or file"` + ServiceAccountName string `default:"cluster-agent" desc:"component service account name"` LeaderElection bool `default:"true" desc:"leader election"` LeaderElectionID string `default:"cluster-agent.erda.cloud" desc:"leader election id"` LeasesResourceLockType string `default:"leases" desc:"leases resource lock type"` diff --git a/internal/tools/cluster-agent/pkg/client/client.go b/internal/tools/cluster-agent/pkg/client/client.go index 67d304a96f2..cf0a13d750b 100644 --- a/internal/tools/cluster-agent/pkg/client/client.go +++ b/internal/tools/cluster-agent/pkg/client/client.go @@ -30,10 +30,22 @@ import ( "github.com/pkg/errors" "github.com/rancher/remotedialer" "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/erda-project/erda/apistructs" "github.com/erda-project/erda/internal/tools/cluster-agent/config" "github.com/erda-project/erda/pkg/discover" + "github.com/erda-project/erda/pkg/k8sclient" +) + +const ( + collectSourceSecret = "secret" + collectSourceFile = "file" +) + +const ( + caCrtKey = "ca.crt" + tokenSecKey = "token" ) const ( @@ -80,7 +92,7 @@ func (c *Client) Start(ctx context.Context) error { } if c.cfg.CollectClusterInfo { - clusterInfo, err := getClusterInfo(c.cfg.K8SApiServerAddr) + clusterInfo, err := c.getClusterInfo() if err != nil { return err } @@ -167,18 +179,55 @@ func (c *Client) IsConnected() bool { return c.connected } -func getClusterInfo(apiServerAddr string) (map[string]interface{}, error) { - caData, err := ioutil.ReadFile(rootCAFile) - if err != nil { - return nil, errors.Wrapf(err, "reading %s", rootCAFile) - } +func (c *Client) getClusterInfo() (map[string]interface{}, error) { + var ( + caData, token []byte + err error + ) - token, err := ioutil.ReadFile(tokenFile) - if err != nil { - return nil, errors.Wrapf(err, "reading %s", tokenFile) + switch c.cfg.CollectSource { + case collectSourceFile: + caData, err = ioutil.ReadFile(rootCAFile) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", rootCAFile) + } + + token, err = ioutil.ReadFile(tokenFile) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", tokenFile) + } + case collectSourceSecret: + k, err := k8sclient.NewForInCluster() + if err != nil { + return nil, err + } + sa, err := k.ClientSet.CoreV1().ServiceAccounts(c.cfg.ErdaNamespace).Get(context.Background(), + c.cfg.ServiceAccountName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + if len(sa.Secrets) == 0 { + return nil, errors.Errorf("service account %s has non auth secret", c.cfg.ServiceAccountName) + } + + saSecret, err := k.ClientSet.CoreV1().Secrets(c.cfg.ErdaNamespace).Get(context.Background(), + sa.Secrets[0].Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + caData = saSecret.Data[caCrtKey] + token = saSecret.Data[tokenSecKey] + default: + return nil, errors.Errorf("collector source %s is illegal", c.cfg.CollectSource) } + + logrus.Debugf("load cluster info, apiserver addr: %s, token: %s, cacert: %s", c.cfg.K8SApiServerAddr, + string(token), string(caData)) + return map[string]interface{}{ - "address": apiServerAddr, + "address": c.cfg.K8SApiServerAddr, "token": strings.TrimSpace(string(token)), "caCert": base64.StdEncoding.EncodeToString(caData), }, nil diff --git a/internal/tools/cluster-agent/pkg/client/client_test.go b/internal/tools/cluster-agent/pkg/client/client_test.go new file mode 100644 index 00000000000..25686720fea --- /dev/null +++ b/internal/tools/cluster-agent/pkg/client/client_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "testing" + + "bou.ke/monkey" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fakeclientset "k8s.io/client-go/kubernetes/fake" + + "github.com/erda-project/erda/internal/tools/cluster-agent/config" + "github.com/erda-project/erda/pkg/k8sclient" +) + +func Test_getClusterInfo(t *testing.T) { + fakeSa := "cluster-agent" + + defer monkey.UnpatchAll() + monkey.Patch(k8sclient.NewForInCluster, + func(...k8sclient.Option) (*k8sclient.K8sClient, error) { + return &k8sclient.K8sClient{ + ClientSet: fakeclientset.NewSimpleClientset(&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: fakeSa, + }, + Secrets: []corev1.ObjectReference{ + { + Name: "cluster-agent-token-mvp6d", + }, + }, + }, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-agent-token-mvp6d", + }, + Data: map[string][]byte{ + caCrtKey: []byte("fake ca data"), + tokenSecKey: []byte("fake token data"), + }, + }), + }, nil + }, + ) + c := New(WithConfig(&config.Config{ + CollectClusterInfo: true, + CollectSource: collectSourceSecret, + ServiceAccountName: fakeSa, + })) + _, err := c.getClusterInfo() + assert.NoError(t, err) +}