Skip to content

Commit

Permalink
fix: cluster agent token expire in kubernetes >= 1.21 (erda-project#5280
Browse files Browse the repository at this point in the history
)
  • Loading branch information
iutx authored Jul 26, 2022
1 parent 48d79aa commit 2d483b0
Show file tree
Hide file tree
Showing 6 changed files with 132 additions and 14 deletions.
2 changes: 2 additions & 0 deletions cmd/cluster-agent/bootstrap.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
cluster-agent:
debug: ${DEBUG:false}
collectSource: "${COLLECT_SOURCE:secret}"
serviceAccountName: "${SERVICE_ACCOUNT_NAME:cluster-agent}"
leaderElection: "${LEADER_ELECTION:true}"
leaderElectionID: "${LEADER_ELECTION_ID:cluster-agent.erda.cloud}"
leasesResourceLockType: "${LEASES_RESOURCE_LOCK_TYPE:leases}"
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ require (
github.com/sirupsen/logrus v1.8.1
github.com/spf13/afero v1.6.0
github.com/spf13/cast v1.3.1
github.com/spf13/cobra v1.4.0
github.com/spf13/cobra v1.5.0
github.com/stretchr/testify v1.7.1
github.com/t-tiger/gorm-bulk-insert v1.3.0
github.com/tealeg/xlsx v1.0.5
Expand Down
6 changes: 3 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawk
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
Expand Down Expand Up @@ -1706,8 +1706,8 @@ github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
Expand Down
2 changes: 2 additions & 0 deletions internal/tools/cluster-agent/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ package config
type Config struct {
Debug bool `default:"false" desc:"enable debug logging"`
CollectClusterInfo bool `default:"true" desc:"enable collect cluster info"`
CollectSource string `default:"secret" desc:"collect source, secret or file"`
ServiceAccountName string `default:"cluster-agent" desc:"component service account name"`
LeaderElection bool `default:"true" desc:"leader election"`
LeaderElectionID string `default:"cluster-agent.erda.cloud" desc:"leader election id"`
LeasesResourceLockType string `default:"leases" desc:"leases resource lock type"`
Expand Down
69 changes: 59 additions & 10 deletions internal/tools/cluster-agent/pkg/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,22 @@ import (
"github.com/pkg/errors"
"github.com/rancher/remotedialer"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/erda-project/erda/apistructs"
"github.com/erda-project/erda/internal/tools/cluster-agent/config"
"github.com/erda-project/erda/pkg/discover"
"github.com/erda-project/erda/pkg/k8sclient"
)

const (
collectSourceSecret = "secret"
collectSourceFile = "file"
)

const (
caCrtKey = "ca.crt"
tokenSecKey = "token"
)

const (
Expand Down Expand Up @@ -80,7 +92,7 @@ func (c *Client) Start(ctx context.Context) error {
}

if c.cfg.CollectClusterInfo {
clusterInfo, err := getClusterInfo(c.cfg.K8SApiServerAddr)
clusterInfo, err := c.getClusterInfo()
if err != nil {
return err
}
Expand Down Expand Up @@ -167,18 +179,55 @@ func (c *Client) IsConnected() bool {
return c.connected
}

func getClusterInfo(apiServerAddr string) (map[string]interface{}, error) {
caData, err := ioutil.ReadFile(rootCAFile)
if err != nil {
return nil, errors.Wrapf(err, "reading %s", rootCAFile)
}
func (c *Client) getClusterInfo() (map[string]interface{}, error) {
var (
caData, token []byte
err error
)

token, err := ioutil.ReadFile(tokenFile)
if err != nil {
return nil, errors.Wrapf(err, "reading %s", tokenFile)
switch c.cfg.CollectSource {
case collectSourceFile:
caData, err = ioutil.ReadFile(rootCAFile)
if err != nil {
return nil, errors.Wrapf(err, "reading %s", rootCAFile)
}

token, err = ioutil.ReadFile(tokenFile)
if err != nil {
return nil, errors.Wrapf(err, "reading %s", tokenFile)
}
case collectSourceSecret:
k, err := k8sclient.NewForInCluster()
if err != nil {
return nil, err
}
sa, err := k.ClientSet.CoreV1().ServiceAccounts(c.cfg.ErdaNamespace).Get(context.Background(),
c.cfg.ServiceAccountName, metav1.GetOptions{})
if err != nil {
return nil, err
}

if len(sa.Secrets) == 0 {
return nil, errors.Errorf("service account %s has non auth secret", c.cfg.ServiceAccountName)
}

saSecret, err := k.ClientSet.CoreV1().Secrets(c.cfg.ErdaNamespace).Get(context.Background(),
sa.Secrets[0].Name, metav1.GetOptions{})
if err != nil {
return nil, err
}

caData = saSecret.Data[caCrtKey]
token = saSecret.Data[tokenSecKey]
default:
return nil, errors.Errorf("collector source %s is illegal", c.cfg.CollectSource)
}

logrus.Debugf("load cluster info, apiserver addr: %s, token: %s, cacert: %s", c.cfg.K8SApiServerAddr,
string(token), string(caData))

return map[string]interface{}{
"address": apiServerAddr,
"address": c.cfg.K8SApiServerAddr,
"token": strings.TrimSpace(string(token)),
"caCert": base64.StdEncoding.EncodeToString(caData),
}, nil
Expand Down
65 changes: 65 additions & 0 deletions internal/tools/cluster-agent/pkg/client/client_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Copyright (c) 2021 Terminus, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
"testing"

"bou.ke/monkey"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
fakeclientset "k8s.io/client-go/kubernetes/fake"

"github.com/erda-project/erda/internal/tools/cluster-agent/config"
"github.com/erda-project/erda/pkg/k8sclient"
)

func Test_getClusterInfo(t *testing.T) {
fakeSa := "cluster-agent"

defer monkey.UnpatchAll()
monkey.Patch(k8sclient.NewForInCluster,
func(...k8sclient.Option) (*k8sclient.K8sClient, error) {
return &k8sclient.K8sClient{
ClientSet: fakeclientset.NewSimpleClientset(&corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: fakeSa,
},
Secrets: []corev1.ObjectReference{
{
Name: "cluster-agent-token-mvp6d",
},
},
}, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-agent-token-mvp6d",
},
Data: map[string][]byte{
caCrtKey: []byte("fake ca data"),
tokenSecKey: []byte("fake token data"),
},
}),
}, nil
},
)
c := New(WithConfig(&config.Config{
CollectClusterInfo: true,
CollectSource: collectSourceSecret,
ServiceAccountName: fakeSa,
}))
_, err := c.getClusterInfo()
assert.NoError(t, err)
}

0 comments on commit 2d483b0

Please sign in to comment.