From 2bf185744261aeabfd5fdb3e3bc6493588510b4a Mon Sep 17 00:00:00 2001 From: l1b0k Date: Mon, 17 Feb 2025 13:29:19 +0800 Subject: [PATCH 1/8] mod: update sdk Signed-off-by: l1b0k --- go.mod | 2 +- go.sum | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a7ccb981..2cdffbf2 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Microsoft/go-winio v0.6.0 github.com/Microsoft/hcsshim v0.9.9 github.com/alexflint/go-filemutex v1.2.0 - github.com/aliyun/alibaba-cloud-sdk-go v1.62.663 + github.com/aliyun/alibaba-cloud-sdk-go v1.63.88 github.com/boltdb/bolt v1.3.1 github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.3.0 diff --git a/go.sum b/go.sum index c43b8b7e..d3ef4ce7 100644 --- a/go.sum +++ b/go.sum @@ -96,8 +96,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/alexflint/go-filemutex v1.2.0 h1:1v0TJPDtlhgpW4nJ+GvxCLSlUDC3+gW0CQQvlmfDR/s= github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.663 h1:LZo0xalmivd0pmvZit+BkH9T2ijzCBnGw0q55CY3AQI= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.663/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o= +github.com/aliyun/alibaba-cloud-sdk-go v1.63.88 h1:87jNTxliGqU2yB3H09xCd4U3cZCmR4AkOMqWgaluo5Q= +github.com/aliyun/alibaba-cloud-sdk-go v1.63.88/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -512,7 +512,6 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -521,7 +520,6 @@ github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52Cu github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -1280,7 +1278,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= From 359d4a15fcc4d38d1bd228e9eb475ee2cc1aa744 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Mon, 17 Feb 2025 13:30:12 +0800 Subject: [PATCH 2/8] crd: add leni Signed-off-by: l1b0k --- pkg/apis/crds/network.alibabacloud.com_nodes.yaml | 4 ++++ pkg/apis/crds/network.alibabacloud.com_podnetworkings.yaml | 4 ++++ pkg/apis/crds/register.go | 2 +- pkg/apis/network.alibabacloud.com/v1beta1/node_types.go | 2 ++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/apis/crds/network.alibabacloud.com_nodes.yaml b/pkg/apis/crds/network.alibabacloud.com_nodes.yaml index 3ddf70a2..a1cce6bb 100644 --- a/pkg/apis/crds/network.alibabacloud.com_nodes.yaml +++ b/pkg/apis/crds/network.alibabacloud.com_nodes.yaml @@ -171,6 +171,8 @@ spec: properties: ip: type: string + ipName: + type: string podID: description: Add the pod ID type: string @@ -199,6 +201,8 @@ spec: properties: ip: type: string + ipName: + type: string podID: description: Add the pod ID type: string diff --git a/pkg/apis/crds/network.alibabacloud.com_podnetworkings.yaml b/pkg/apis/crds/network.alibabacloud.com_podnetworkings.yaml index 0011cd30..78c8772c 100644 --- a/pkg/apis/crds/network.alibabacloud.com_podnetworkings.yaml +++ b/pkg/apis/crds/network.alibabacloud.com_podnetworkings.yaml @@ -112,11 +112,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -159,11 +161,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string diff --git a/pkg/apis/crds/register.go b/pkg/apis/crds/register.go index fad9f2a4..93361ed7 100644 --- a/pkg/apis/crds/register.go +++ b/pkg/apis/crds/register.go @@ -47,7 +47,7 @@ func getCRD(name string) apiextensionsv1.CustomResourceDefinition { switch name { case CRDPodENI: crdBytes = crdsPodENI - version = "v0.2.0" + version = "v0.3.0" case CRDPodNetworking: crdBytes = crdsPodNetworking version = "v0.1.0" diff --git a/pkg/apis/network.alibabacloud.com/v1beta1/node_types.go b/pkg/apis/network.alibabacloud.com/v1beta1/node_types.go index 5d157f68..37d069ea 100644 --- a/pkg/apis/network.alibabacloud.com/v1beta1/node_types.go +++ b/pkg/apis/network.alibabacloud.com/v1beta1/node_types.go @@ -124,6 +124,8 @@ type IP struct { PodID string `json:"podID,omitempty"` // Add pod UID for validate PodUID string `json:"podUID,omitempty"` + + IPName string `json:"ipName,omitempty"` } type IPMap map[string]*IP From 8668da416807455b72629093122321ce1ad11328 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Mon, 17 Feb 2025 13:40:13 +0800 Subject: [PATCH 3/8] client: support endpoint type Signed-off-by: l1b0k --- pkg/aliyun/credential/aliyun_client_mgr.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/aliyun/credential/aliyun_client_mgr.go b/pkg/aliyun/credential/aliyun_client_mgr.go index 15b6bc38..5be96410 100644 --- a/pkg/aliyun/credential/aliyun_client_mgr.go +++ b/pkg/aliyun/credential/aliyun_client_mgr.go @@ -85,6 +85,8 @@ type ClientMgr struct { ecsDomainOverride string vpcDomainOverride string efloDomainOverride string + + endpointType string } // NewClientMgr return new aliyun client manager @@ -106,6 +108,12 @@ func NewClientMgr(regionID string, providers ...Interface) (*ClientMgr, error) { if err != nil { return nil, err } + + mgr.endpointType = "vpc" + if os.Getenv("ALICLOUD_ENDPOINT_TYPE") != "" { + mgr.endpointType = os.Getenv("ALICLOUD_ENDPOINT_TYPE") + } + for _, p := range providers { c, err := p.Resolve() if err != nil { @@ -181,7 +189,7 @@ func (c *ClientMgr) refreshToken() (bool, error) { if err != nil { return false, err } - c.ecs.SetEndpointRules(c.ecs.EndpointMap, "regional", "vpc") + c.ecs.SetEndpointRules(c.ecs.EndpointMap, "regional", c.endpointType) if c.ecsDomainOverride != "" { c.ecs.Domain = c.ecsDomainOverride @@ -191,7 +199,7 @@ func (c *ClientMgr) refreshToken() (bool, error) { if err != nil { return false, err } - c.vpc.SetEndpointRules(c.vpc.EndpointMap, "regional", "vpc") + c.vpc.SetEndpointRules(c.vpc.EndpointMap, "regional", c.endpointType) if c.vpcDomainOverride != "" { c.vpc.Domain = c.vpcDomainOverride @@ -201,7 +209,7 @@ func (c *ClientMgr) refreshToken() (bool, error) { if err != nil { return false, err } - c.eflo.SetEndpointRules(c.eflo.EndpointMap, "regional", "vpc") + c.eflo.SetEndpointRules(c.eflo.EndpointMap, "regional", c.endpointType) if c.efloDomainOverride != "" { c.eflo.Domain = c.efloDomainOverride From 75771893c7e6712565c791036d533419a0dff8b2 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Mon, 17 Feb 2025 14:37:07 +0800 Subject: [PATCH 4/8] api: add v2 eflo api Signed-off-by: l1b0k --- pkg/aliyun/client/ecs_options.go | 175 ++++++++++++++++++++++ pkg/aliyun/client/interface_default.go | 34 +++++ pkg/aliyun/client/types.go | 87 +++++++---- pkg/controller/multi-ip/node/eni.go | 28 ++-- pkg/controller/multi-ip/node/pool_test.go | 75 +++++----- pkg/controller/pod/pod_controller.go | 2 +- pkg/factory/aliyun/aliyun.go | 4 +- 7 files changed, 322 insertions(+), 83 deletions(-) diff --git a/pkg/aliyun/client/ecs_options.go b/pkg/aliyun/client/ecs_options.go index 92fd3b60..5608bbb8 100644 --- a/pkg/aliyun/client/ecs_options.go +++ b/pkg/aliyun/client/ecs_options.go @@ -3,6 +3,7 @@ package client import ( "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/aliyun/alibaba-cloud-sdk-go/services/eflo" "k8s.io/apimachinery/pkg/util/wait" ) @@ -220,3 +221,177 @@ func (c *AssignIPv6AddressesOptions) Finish(idempotentKeyGen IdempotentKeyGen) ( idempotentKeyGen.PutBack(argsHash, req.ClientToken) }, nil } + +type DescribeNetworkInterfaceOption interface { + ApplyTo(*DescribeNetworkInterfaceOptions) +} + +type DescribeNetworkInterfaceOptions struct { + VPCID *string + NetworkInterfaceIDs *[]string + InstanceID *string + InstanceType *string + Status *string + Tags *map[string]string + + Backoff *wait.Backoff +} + +func (o *DescribeNetworkInterfaceOptions) ApplyTo(in *DescribeNetworkInterfaceOptions) { + if o.VPCID != nil { + in.VPCID = o.VPCID + } + if o.NetworkInterfaceIDs != nil { + in.NetworkInterfaceIDs = o.NetworkInterfaceIDs + } + if o.InstanceID != nil { + in.InstanceID = o.InstanceID + } + if o.InstanceType != nil { + in.InstanceType = o.InstanceType + } + if o.Status != nil { + in.Status = o.Status + } + if o.Tags != nil { + in.Tags = o.Tags + } + if o.Backoff != nil { + in.Backoff = o.Backoff + } +} + +func (o *DescribeNetworkInterfaceOptions) ECS() *ecs.DescribeNetworkInterfacesRequest { + req := ecs.CreateDescribeNetworkInterfacesRequest() + if o.VPCID != nil { + req.VpcId = *o.VPCID + } + if o.NetworkInterfaceIDs != nil { + req.NetworkInterfaceId = o.NetworkInterfaceIDs + } + if o.InstanceID != nil { + req.InstanceId = *o.InstanceID + } + if o.InstanceType != nil { + req.Type = *o.InstanceType + } + if o.Status != nil { + req.Status = *o.Status + } + if o.Tags != nil { + tags := make([]ecs.DescribeNetworkInterfacesTag, 0) + for k, v := range *o.Tags { + tags = append(tags, ecs.DescribeNetworkInterfacesTag{ + Key: k, + Value: v, + }) + } + req.Tag = &tags + } + + if o.Backoff == nil { + o.Backoff = &wait.Backoff{ + Steps: 1, + } + } + + return req +} + +func (o *DescribeNetworkInterfaceOptions) EFLO() *eflo.ListElasticNetworkInterfacesRequest { + req := eflo.CreateListElasticNetworkInterfacesRequest() + if o.VPCID != nil { + req.VpcId = *o.VPCID + } + + if o.NetworkInterfaceIDs != nil && len(*o.NetworkInterfaceIDs) > 0 { + req.ElasticNetworkInterfaceId = (*o.NetworkInterfaceIDs)[0] + } + if o.InstanceID != nil { + req.NodeId = *o.InstanceID + } + if o.InstanceType != nil { + req.Type = *o.InstanceType + } + if o.Status != nil { + req.Status = *o.Status + } + + if o.Backoff == nil { + o.Backoff = &wait.Backoff{ + Steps: 1, + } + } + + return req +} + +type AttachNetworkInterfaceOption interface { + ApplyTo(*AttachNetworkInterfaceOptions) +} + +type AttachNetworkInterfaceOptions struct { + NetworkInterfaceID, InstanceID, TrunkNetworkInstanceId *string +} + +func (o *AttachNetworkInterfaceOptions) ApplyTo(in *AttachNetworkInterfaceOptions) { + if o.NetworkInterfaceID != nil { + in.NetworkInterfaceID = o.NetworkInterfaceID + } + if o.InstanceID != nil { + in.InstanceID = o.InstanceID + } + if o.TrunkNetworkInstanceId != nil { + in.TrunkNetworkInstanceId = o.TrunkNetworkInstanceId + } +} + +func (o *AttachNetworkInterfaceOptions) ECS() (*ecs.AttachNetworkInterfaceRequest, error) { + if o.NetworkInterfaceID == nil || o.InstanceID == nil { + return nil, ErrInvalidArgs + } + req := ecs.CreateAttachNetworkInterfaceRequest() + req.NetworkInterfaceId = *o.NetworkInterfaceID + req.InstanceId = *o.InstanceID + + if o.TrunkNetworkInstanceId != nil { + req.TrunkNetworkInstanceId = *o.TrunkNetworkInstanceId + } + + return req, nil +} + +type DetachNetworkInterfaceOption interface { + ApplyTo(*DetachNetworkInterfaceOptions) +} + +type DetachNetworkInterfaceOptions struct { + NetworkInterfaceID, InstanceID, TrunkENIID *string +} + +func (o *DetachNetworkInterfaceOptions) ApplyTo(in *DetachNetworkInterfaceOptions) { + if o.NetworkInterfaceID != nil { + in.NetworkInterfaceID = o.NetworkInterfaceID + } + if o.InstanceID != nil { + in.InstanceID = o.InstanceID + } + if o.TrunkENIID != nil { + in.TrunkENIID = o.TrunkENIID + } +} + +func (o *DetachNetworkInterfaceOptions) ECS() (*ecs.DetachNetworkInterfaceRequest, error) { + if o.NetworkInterfaceID == nil || o.InstanceID == nil { + return nil, ErrInvalidArgs + } + req := ecs.CreateDetachNetworkInterfaceRequest() + req.NetworkInterfaceId = *o.NetworkInterfaceID + req.InstanceId = *o.InstanceID + + if o.TrunkENIID != nil { + req.TrunkNetworkInstanceId = *o.TrunkENIID + } + + return req, nil +} diff --git a/pkg/aliyun/client/interface_default.go b/pkg/aliyun/client/interface_default.go index 1da0d363..66887524 100644 --- a/pkg/aliyun/client/interface_default.go +++ b/pkg/aliyun/client/interface_default.go @@ -25,3 +25,37 @@ type ECS interface { UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []netip.Addr) error DescribeInstanceTypes(ctx context.Context, types []string) ([]ecs.InstanceType, error) } + +type BackendAPI int + +const ( + BackendAPIECS BackendAPI = iota + BackendAPIEFLO +) + +type backendAPIKey struct{} + +func GetBackendAPI(ctx context.Context) BackendAPI { + value, ok := ctx.Value(backendAPIKey{}).(BackendAPI) + if !ok { + return BackendAPIECS + } + return value +} + +func SetBackendAPI(ctx context.Context, b BackendAPI) context.Context { + return context.WithValue(ctx, backendAPIKey{}, b) +} + +type ENI interface { + CreateNetworkInterfaceV2(ctx context.Context, opts ...CreateNetworkInterfaceOption) (*NetworkInterface, error) + DescribeNetworkInterfaceV2(ctx context.Context, opts ...DescribeNetworkInterfaceOption) ([]*NetworkInterface, error) + AttachNetworkInterfaceV2(ctx context.Context, opts ...AttachNetworkInterfaceOption) error + DetachNetworkInterfaceV2(ctx context.Context, opts ...DetachNetworkInterfaceOption) error + DeleteNetworkInterfaceV2(ctx context.Context, eniID string) error + AssignPrivateIPAddressV2(ctx context.Context, opts ...AssignPrivateIPAddressOption) ([]IPSet, error) + UnAssignPrivateIPAddressesV2(ctx context.Context, eniID string, ips []IPSet) error + AssignIpv6AddressesV2(ctx context.Context, opts ...AssignIPv6AddressesOption) ([]IPSet, error) + UnAssignIpv6AddressesV2(ctx context.Context, eniID string, ips []IPSet) error + WaitForNetworkInterfaceV2(ctx context.Context, eniID string, status string, backoff wait.Backoff, ignoreNotExist bool) (*NetworkInterface, error) +} diff --git a/pkg/aliyun/client/types.go b/pkg/aliyun/client/types.go index b7771b3f..6d6904a5 100644 --- a/pkg/aliyun/client/types.go +++ b/pkg/aliyun/client/types.go @@ -7,6 +7,7 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/go-logr/logr" + "github.com/samber/lo" ) var ErrInvalidArgs = errors.New("invalid args") @@ -40,6 +41,10 @@ const ( ENIStatusDeleting string = "Deleting" ) +const ( + LENIStatusAvailable string = "Available" +) + const ( ENITypePrimary string = "Primary" ENITypeSecondary string = "Secondary" @@ -56,17 +61,18 @@ const EIPInstanceTypeNetworkInterface = "NetworkInterface" // NetworkInterface openAPI result for ecs.CreateNetworkInterfaceResponse and ecs.NetworkInterfaceSet type NetworkInterface struct { - Status string `json:"status,omitempty"` - MacAddress string `json:"mac_address,omitempty"` - NetworkInterfaceID string `json:"network_interface_id,omitempty"` - VSwitchID string `json:"v_switch_id,omitempty"` - PrivateIPAddress string `json:"private_ip_address,omitempty"` - PrivateIPSets []ecs.PrivateIpSet `json:"private_ip_sets"` - ZoneID string `json:"zone_id,omitempty"` - SecurityGroupIDs []string `json:"security_group_ids,omitempty"` - ResourceGroupID string `json:"resource_group_id,omitempty"` - IPv6Set []ecs.Ipv6Set `json:"ipv6_set,omitempty"` - Tags []ecs.Tag `json:"tags,omitempty"` + Status string `json:"status,omitempty"` + MacAddress string `json:"mac_address,omitempty"` + NetworkInterfaceID string `json:"network_interface_id,omitempty"` + VPCID string `json:"vpc_ic,omitempty"` + VSwitchID string `json:"v_switch_id,omitempty"` + PrivateIPAddress string `json:"private_ip_address,omitempty"` + PrivateIPSets []IPSet `json:"private_ip_sets"` + ZoneID string `json:"zone_id,omitempty"` + SecurityGroupIDs []string `json:"security_group_ids,omitempty"` + ResourceGroupID string `json:"resource_group_id,omitempty"` + IPv6Set []IPSet `json:"ipv6_set,omitempty"` + Tags []ecs.Tag `json:"tags,omitempty"` // fields for DescribeNetworkInterface Type string `json:"type,omitempty"` @@ -77,6 +83,13 @@ type NetworkInterface struct { CreationTime string `json:"creation_time,omitempty"` } +type IPSet struct { + Primary bool + IPAddress string + IPName string + IPStatus string +} + func FromCreateResp(in *ecs.CreateNetworkInterfaceResponse) *NetworkInterface { return &NetworkInterface{ Status: in.Status, @@ -84,13 +97,22 @@ func FromCreateResp(in *ecs.CreateNetworkInterfaceResponse) *NetworkInterface { NetworkInterfaceID: in.NetworkInterfaceId, VSwitchID: in.VSwitchId, PrivateIPAddress: in.PrivateIpAddress, - PrivateIPSets: in.PrivateIpSets.PrivateIpSet, - ZoneID: in.ZoneId, - SecurityGroupIDs: in.SecurityGroupIds.SecurityGroupId, - IPv6Set: in.Ipv6Sets.Ipv6Set, - Tags: in.Tags.Tag, - Type: in.Type, - ResourceGroupID: in.ResourceGroupId, + PrivateIPSets: lo.Map(in.PrivateIpSets.PrivateIpSet, func(item ecs.PrivateIpSet, _ int) IPSet { + return IPSet{ + IPAddress: item.PrivateIpAddress, + Primary: item.Primary, + } + }), + ZoneID: in.ZoneId, + SecurityGroupIDs: in.SecurityGroupIds.SecurityGroupId, + IPv6Set: lo.Map(in.Ipv6Sets.Ipv6Set, func(item ecs.Ipv6Set, _ int) IPSet { + return IPSet{ + IPAddress: item.Ipv6Address, + } + }), + Tags: in.Tags.Tag, + Type: in.Type, + ResourceGroupID: in.ResourceGroupId, } } @@ -101,16 +123,25 @@ func FromDescribeResp(in *ecs.NetworkInterfaceSet) *NetworkInterface { } return &NetworkInterface{ - Status: in.Status, - MacAddress: in.MacAddress, - NetworkInterfaceID: in.NetworkInterfaceId, - InstanceID: ins, - VSwitchID: in.VSwitchId, - PrivateIPAddress: in.PrivateIpAddress, - ZoneID: in.ZoneId, - SecurityGroupIDs: in.SecurityGroupIds.SecurityGroupId, - IPv6Set: in.Ipv6Sets.Ipv6Set, - PrivateIPSets: in.PrivateIpSets.PrivateIpSet, + Status: in.Status, + MacAddress: in.MacAddress, + NetworkInterfaceID: in.NetworkInterfaceId, + InstanceID: ins, + VSwitchID: in.VSwitchId, + PrivateIPAddress: in.PrivateIpAddress, + ZoneID: in.ZoneId, + SecurityGroupIDs: in.SecurityGroupIds.SecurityGroupId, + IPv6Set: lo.Map(in.Ipv6Sets.Ipv6Set, func(item ecs.Ipv6Set, _ int) IPSet { + return IPSet{ + IPAddress: item.Ipv6Address, + } + }), + PrivateIPSets: lo.Map(in.PrivateIpSets.PrivateIpSet, func(item ecs.PrivateIpSet, _ int) IPSet { + return IPSet{ + IPAddress: item.PrivateIpAddress, + Primary: item.Primary, + } + }), Tags: in.Tags.Tag, TrunkNetworkInterfaceID: in.Attachment.TrunkNetworkInterfaceId, NetworkInterfaceTrafficMode: in.NetworkInterfaceTrafficMode, diff --git a/pkg/controller/multi-ip/node/eni.go b/pkg/controller/multi-ip/node/eni.go index a62a25ca..3d3fdc61 100644 --- a/pkg/controller/multi-ip/node/eni.go +++ b/pkg/controller/multi-ip/node/eni.go @@ -3,7 +3,6 @@ package node import ( "sort" - "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/go-logr/logr" "github.com/samber/lo" @@ -130,22 +129,23 @@ func newENIFromAPI(eni *aliyunClient.NetworkInterface) *networkv1beta1.NetworkIn PrimaryIPAddress: eni.PrivateIPAddress, NetworkInterfaceTrafficMode: networkv1beta1.NetworkInterfaceTrafficMode(eni.NetworkInterfaceTrafficMode), NetworkInterfaceType: networkv1beta1.ENIType(eni.Type), - IPv4: lo.SliceToMap(eni.PrivateIPSets, func(item ecs.PrivateIpSet) (string, *networkv1beta1.IP) { - return item.PrivateIpAddress, &networkv1beta1.IP{ - IP: item.PrivateIpAddress, - Status: networkv1beta1.IPStatusValid, - Primary: item.Primary, - } - }), - IPv6: lo.SliceToMap(eni.IPv6Set, func(item ecs.Ipv6Set) (string, *networkv1beta1.IP) { - return item.Ipv6Address, &networkv1beta1.IP{ - IP: item.Ipv6Address, - Status: networkv1beta1.IPStatusValid, - } - }), + IPv4: convertIPSet(eni.PrivateIPSets), + IPv6: convertIPSet(eni.IPv6Set), } } +// convertIPSet convert aliyunClient.IPSet to networkv1beta1.IP +// for eflo case need to handle the ip status here +func convertIPSet(in []aliyunClient.IPSet) map[string]*networkv1beta1.IP { + return lo.SliceToMap(in, func(item aliyunClient.IPSet) (string, *networkv1beta1.IP) { + return item.IPAddress, &networkv1beta1.IP{ + IP: item.IPAddress, + Status: networkv1beta1.IPStatusValid, + Primary: item.Primary, + } + }) +} + func mergeIPMap(log logr.Logger, remote, current map[string]*networkv1beta1.IP) { // delete remote not in current for k := range current { diff --git a/pkg/controller/multi-ip/node/pool_test.go b/pkg/controller/multi-ip/node/pool_test.go index 6cdf71c6..e4f2c2c8 100644 --- a/pkg/controller/multi-ip/node/pool_test.go +++ b/pkg/controller/multi-ip/node/pool_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" @@ -440,14 +439,14 @@ func TestReconcileNodeSyncWithAPI(t *testing.T) { NetworkInterfaceID: "eni-3", VSwitchID: "vsw-1", PrivateIPAddress: "", - PrivateIPSets: []ecs.PrivateIpSet{ + PrivateIPSets: []aliyunClient.IPSet{ { - PrivateIpAddress: "192.168.0.1", - Primary: true, + IPAddress: "192.168.0.1", + Primary: true, }, { - PrivateIpAddress: "192.168.0.2", - Primary: false, + IPAddress: "192.168.0.2", + Primary: false, }, }, ZoneID: "zone-1", @@ -1014,25 +1013,25 @@ func TestReconcileNode_createENI(t *testing.T) { NetworkInterfaceID: "eni-1", VSwitchID: "vsw-1", PrivateIPAddress: "127.0.0.1", - PrivateIPSets: []ecs.PrivateIpSet{ + PrivateIPSets: []aliyunClient.IPSet{ { - PrivateIpAddress: "127.0.0.1", - Primary: true, + IPAddress: "127.0.0.1", + Primary: true, }, { - PrivateIpAddress: "127.0.0.2", - Primary: false, + IPAddress: "127.0.0.2", + Primary: false, }, }, ZoneID: "zone-1", SecurityGroupIDs: nil, ResourceGroupID: "", - IPv6Set: []ecs.Ipv6Set{ + IPv6Set: []aliyunClient.IPSet{ { - Ipv6Address: "fd00::1", + IPAddress: "fd00::1", }, { - Ipv6Address: "fd00::2", + IPAddress: "fd00::2", }, }, Tags: nil, @@ -1047,25 +1046,25 @@ func TestReconcileNode_createENI(t *testing.T) { NetworkInterfaceID: "eni-1", VSwitchID: "vsw-1", PrivateIPAddress: "127.0.0.1", - PrivateIPSets: []ecs.PrivateIpSet{ + PrivateIPSets: []aliyunClient.IPSet{ { - PrivateIpAddress: "127.0.0.1", - Primary: true, + IPAddress: "127.0.0.1", + Primary: true, }, { - PrivateIpAddress: "127.0.0.2", - Primary: false, + IPAddress: "127.0.0.2", + Primary: false, }, }, ZoneID: "zone-1", SecurityGroupIDs: nil, ResourceGroupID: "", - IPv6Set: []ecs.Ipv6Set{ + IPv6Set: []aliyunClient.IPSet{ { - Ipv6Address: "fd00::1", + IPAddress: "fd00::1", }, { - Ipv6Address: "fd00::2", + IPAddress: "fd00::2", }, }, Tags: nil, @@ -1140,25 +1139,25 @@ func TestReconcileNode_createENI(t *testing.T) { NetworkInterfaceID: "eni-1", VSwitchID: "vsw-1", PrivateIPAddress: "127.0.0.1", - PrivateIPSets: []ecs.PrivateIpSet{ + PrivateIPSets: []aliyunClient.IPSet{ { - PrivateIpAddress: "127.0.0.1", - Primary: true, + IPAddress: "127.0.0.1", + Primary: true, }, { - PrivateIpAddress: "127.0.0.2", - Primary: false, + IPAddress: "127.0.0.2", + Primary: false, }, }, ZoneID: "zone-1", SecurityGroupIDs: nil, ResourceGroupID: "", - IPv6Set: []ecs.Ipv6Set{ + IPv6Set: []aliyunClient.IPSet{ { - Ipv6Address: "fd00::1", + IPAddress: "fd00::1", }, { - Ipv6Address: "fd00::2", + IPAddress: "fd00::2", }, }, Tags: nil, @@ -1666,25 +1665,25 @@ func TestReconcileNode_handleStatus(t *testing.T) { NetworkInterfaceID: "eni-1", VSwitchID: "vsw-1", PrivateIPAddress: "127.0.0.1", - PrivateIPSets: []ecs.PrivateIpSet{ + PrivateIPSets: []aliyunClient.IPSet{ { - PrivateIpAddress: "127.0.0.1", - Primary: true, + IPAddress: "127.0.0.1", + Primary: true, }, { - PrivateIpAddress: "127.0.0.2", - Primary: false, + IPAddress: "127.0.0.2", + Primary: false, }, }, ZoneID: "zone-1", SecurityGroupIDs: nil, ResourceGroupID: "", - IPv6Set: []ecs.Ipv6Set{ + IPv6Set: []aliyunClient.IPSet{ { - Ipv6Address: "fd00::1", + IPAddress: "fd00::1", }, { - Ipv6Address: "fd00::2", + IPAddress: "fd00::2", }, }, Tags: nil, diff --git a/pkg/controller/pod/pod_controller.go b/pkg/controller/pod/pod_controller.go index b26ba8ea..89aa4744 100644 --- a/pkg/controller/pod/pod_controller.go +++ b/pkg/controller/pod/pod_controller.go @@ -585,7 +585,7 @@ func (m *ReconcilePod) createENI(ctx context.Context, allocs *[]*v1beta1.Allocat v6 := "" if len(eni.IPv6Set) > 0 { - v6 = eni.IPv6Set[0].Ipv6Address + v6 = eni.IPv6Set[0].IPAddress } alloc.ENI = v1beta1.ENI{ ID: eni.NetworkInterfaceID, diff --git a/pkg/factory/aliyun/aliyun.go b/pkg/factory/aliyun/aliyun.go index 5dd9d522..bff22c97 100644 --- a/pkg/factory/aliyun/aliyun.go +++ b/pkg/factory/aliyun/aliyun.go @@ -148,7 +148,7 @@ func (a *Aliyun) CreateNetworkInterface(ipv4, ipv6 int, eniType string) (*daemon v4Set, err := func() ([]netip.Addr, error) { var ips []netip.Addr for _, v := range eni.PrivateIPSets { - addr, err := netip.ParseAddr(v.PrivateIpAddress) + addr, err := netip.ParseAddr(v.IPAddress) if err != nil { return nil, err } @@ -163,7 +163,7 @@ func (a *Aliyun) CreateNetworkInterface(ipv4, ipv6 int, eniType string) (*daemon v6Set, err := func() ([]netip.Addr, error) { var ips []netip.Addr for _, v := range eni.IPv6Set { - addr, err := netip.ParseAddr(v.Ipv6Address) + addr, err := netip.ParseAddr(v.IPAddress) if err != nil { return nil, err } From cc5faf18059cfc87de464f1a8738d26f53f202f4 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Mon, 17 Feb 2025 15:56:22 +0800 Subject: [PATCH 5/8] api: impl the eflo api Signed-off-by: l1b0k --- pkg/aliyun/client/eflo.go | 43 +++ pkg/aliyun/client/eflo_2.go | 300 ++++++++++++++++++ pkg/aliyun/client/eni.go | 110 +++++++ pkg/aliyun/client/evc_2.go | 268 ++++++++++++++++ pkg/aliyun/client/interface_default.go | 4 +- .../client/{ecs_options.go => options.go} | 146 +++++---- .../{ecs_options_test.go => options_test.go} | 0 pkg/aliyun/client/types.go | 3 +- pkg/controller/node/predict.go | 2 +- pkg/utils/k8s.go | 4 +- 10 files changed, 804 insertions(+), 76 deletions(-) create mode 100644 pkg/aliyun/client/eflo_2.go create mode 100644 pkg/aliyun/client/eni.go create mode 100644 pkg/aliyun/client/evc_2.go rename pkg/aliyun/client/{ecs_options.go => options.go} (82%) rename pkg/aliyun/client/{ecs_options_test.go => options_test.go} (100%) diff --git a/pkg/aliyun/client/eflo.go b/pkg/aliyun/client/eflo.go index 2fd90c9b..6b930719 100644 --- a/pkg/aliyun/client/eflo.go +++ b/pkg/aliyun/client/eflo.go @@ -28,6 +28,10 @@ func (a *OpenAPI) CreateElasticNetworkInterface(zoneID, nodeID, vSwitchID, secur return "", "", err } + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + return "", "", err + } return resp.Content.NodeId, resp.Content.ElasticNetworkInterfaceId, nil } @@ -48,6 +52,12 @@ func (a *OpenAPI) DeleteElasticNetworkInterface(ctx context.Context, eniID strin return err } + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + l.Error(err, "failed") + return err + } + l.WithValues(LogFieldRequestID, resp.RequestId).Info("succeed") return nil } @@ -72,6 +82,11 @@ func (a *OpenAPI) AssignLeniPrivateIPAddress(ctx context.Context, eniID, prefer return "", err } + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + l.Error(err, "failed") + return "", err + } l.WithValues(LogFieldRequestID, resp.RequestId).Info("assign", "ipName", resp.Content.IpName, "ip", resp.Content.Ip, "private", resp.Content.PrivateIpAddress) return resp.Content.IpName, nil @@ -97,6 +112,11 @@ func (a *OpenAPI) UnassignLeniPrivateIPAddress(ctx context.Context, eniID, ipNam return err } + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + l.Error(err, "failed") + return err + } l.WithValues(LogFieldRequestID, resp.RequestId).Info("success") return nil @@ -113,6 +133,11 @@ func (a *OpenAPI) GetElasticNetworkInterface(eniID string) (*eflo.Content, error return nil, err } + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + return nil, err + } + return &resp.Content, nil } @@ -138,6 +163,12 @@ func (a *OpenAPI) ListLeniPrivateIPAddresses(ctx context.Context, eniID, ipName, return nil, err } + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + l.Error(err, "failed") + return nil, err + } + l.WithValues(LogFieldRequestID, resp.RequestId).Info("success") return &resp.Content, nil @@ -164,6 +195,12 @@ func (a *OpenAPI) ListElasticNetworkInterfaces(ctx context.Context, zoneID, node return nil, err } + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + l.Error(err, "failed") + return nil, err + } + l.WithValues(LogFieldRequestID, resp.RequestId).Info("success") return &resp.Content, nil @@ -186,5 +223,11 @@ func (a *OpenAPI) GetNodeInfoForPod(ctx context.Context, nodeID string) (*eflo.C } l.WithValues(LogFieldRequestID, resp.RequestId).Info("success") + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + l.Error(err, "failed") + return nil, err + } + return &resp.Content, nil } diff --git a/pkg/aliyun/client/eflo_2.go b/pkg/aliyun/client/eflo_2.go new file mode 100644 index 00000000..3ccc30d2 --- /dev/null +++ b/pkg/aliyun/client/eflo_2.go @@ -0,0 +1,300 @@ +package client + +import ( + "context" + "fmt" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/services/eflo" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + + apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" + "github.com/AliyunContainerService/terway/pkg/metric" +) + +const ( + APICreateElasticNetworkInterface = "CreateElasticNetworkInterface" + APIAssignLeniPrivateIPAddress = "AssignLeniPrivateIpAddress" + APIDeleteElasticNetworkInterface = "DeleteElasticNetworkInterface" + APIUnassignLeniPrivateIPAddress = "UnassignLeniPrivateIpAddress" + APIListLeniPrivateIPAddresses = "ListLeniPrivateIpAddresses" + APIListElasticNetworkInterfaces = "ListElasticNetworkInterfaces" + APIGetNodeInfoForPod = "GetNodeInfoForPod" +) + +func (a *OpenAPI) CreateElasticNetworkInterfaceV2(ctx context.Context, opts ...CreateNetworkInterfaceOption) (*NetworkInterface, error) { + options := &CreateNetworkInterfaceOptions{} + for _, opt := range opts { + opt.ApplyCreateNetworkInterface(options) + } + + req, rollBackFunc, err := options.EFLO(a.IdempotentKeyGen) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + rollBackFunc() + } + }() + + err = a.RateLimiter.Wait(ctx, APICreateElasticNetworkInterface) + if err != nil { + return nil, err + } + + l := LogFields(logr.FromContextOrDiscard(ctx), req) + + start := time.Now() + resp, err := a.ClientSet.EFLO().CreateElasticNetworkInterface(req) + metric.OpenAPILatency.WithLabelValues("CreateElasticNetworkInterface", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) + if err != nil { + return nil, err + } + + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + l.Error(err, "failed") + return nil, err + } + + l.WithValues(LogFieldRequestID, resp.RequestId).Info("leni created") + + return &NetworkInterface{ + NetworkInterfaceID: resp.Content.ElasticNetworkInterfaceId, + }, err +} + +func (a *OpenAPI) DescribeLeniNetworkInterface(ctx context.Context, opts ...DescribeNetworkInterfaceOption) ([]*NetworkInterface, error) { + options := &DescribeNetworkInterfaceOptions{} + for _, opt := range opts { + opt.ApplyTo(options) + } + req := options.EFLO() + + l := LogFields(logr.FromContextOrDiscard(ctx), req) + + resp, err := a.ClientSet.EFLO().ListElasticNetworkInterfaces(req) + if err != nil { + return nil, err + } + if resp.Code != 0 { + err = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + l.Error(err, "failed") + return nil, err + } + enis := make([]*NetworkInterface, 0) + for _, data := range resp.Content.Data { + + logr.FromContextOrDiscard(ctx).Info("ListElasticNetworkInterfaces", "data", data) + + if data.Type == "DEFAULT" { // CUSTOM for our own card + continue + } + + var privateIPs []IPSet + eni := &NetworkInterface{ + Status: data.Status, + MacAddress: data.Mac, + NetworkInterfaceID: data.ElasticNetworkInterfaceId, + VSwitchID: data.VSwitchId, + VPCID: data.VpcId, + PrivateIPAddress: data.Ip, + PrivateIPSets: []IPSet{ + { + IPAddress: data.Ip, + Primary: true, + }, + }, + ZoneID: data.ZoneId, + SecurityGroupIDs: []string{data.SecurityGroupId}, + ResourceGroupID: data.ResourceGroupId, + IPv6Set: nil, + Tags: nil, + Type: ENITypeSecondary, + InstanceID: data.NodeId, + TrunkNetworkInterfaceID: "", + NetworkInterfaceTrafficMode: ENITrafficModeStandard, + DeviceIndex: 0, + } + + // For now use ecs status + switch eni.Status { + case LENIStatusUnattached: + eni.Status = ENIStatusAvailable + case LENIStatusAvailable: + eni.Status = ENIStatusInUse + } + + privateIPs = append(privateIPs, IPSet{ + IPAddress: data.Ip, + Primary: true, // primary will not hav ipname + }) + + ipsData, err := a.ListLeniPrivateIPAddresses(ctx, eni.NetworkInterfaceID, "", "") + if err != nil { + return nil, err + } + + for _, ips := range ipsData.Data { + privateIPs = append(privateIPs, IPSet{ + IPAddress: ips.PrivateIpAddress, + Primary: false, + IPName: ips.IpName, + IPStatus: ips.Status, + }) + } + + eni.PrivateIPSets = privateIPs + enis = append(enis, eni) + } + return enis, nil +} + +func (a *OpenAPI) AssignLeniPrivateIPAddress2(ctx context.Context, opts ...AssignPrivateIPAddressOption) ([]IPSet, error) { + ctx, span := a.Tracer.Start(ctx, APIAssignLeniPrivateIPAddress) + defer span.End() + + option := &AssignPrivateIPAddressOptions{} + for _, opt := range opts { + opt.ApplyAssignPrivateIPAddress(option) + } + + req, rollBackFunc, err := option.EFLO(a.IdempotentKeyGen) + if err != nil { + return nil, err + } + l := LogFields(logr.FromContextOrDiscard(ctx), req) + + var ( + resp *eflo.AssignLeniPrivateIpAddressResponse + innerErr error + ) + + err = wait.ExponentialBackoffWithContext(ctx, *option.Backoff, func(ctx context.Context) (bool, error) { + innerErr = a.RateLimiter.Wait(ctx, APIAssignLeniPrivateIPAddress) + if innerErr != nil { + return true, innerErr + } + start := time.Now() + resp, innerErr = a.ClientSet.EFLO().AssignLeniPrivateIpAddress(req) + metric.OpenAPILatency.WithLabelValues(APIAssignLeniPrivateIPAddress, fmt.Sprint(innerErr != nil)).Observe(metric.MsSince(start)) + + if innerErr != nil { + innerErr = apiErr.WarpError(innerErr) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(innerErr)).Error(innerErr, "failed") + if apiErr.ErrorIs(innerErr, apiErr.IsURLError, apiErr.WarpFn(apiErr.ErrThrottling, apiErr.ErrInternalError, apiErr.ErrOperationConflict)) { + return false, nil + } + return true, innerErr + } + + if resp.Code != 0 { + innerErr = fmt.Errorf("%s requestID %s", resp.Message, resp.RequestId) + l.Error(err, "failed") + return true, innerErr + } + + return true, nil + }) + if err != nil { + rollBackFunc() + return nil, err + } + + ipName := resp.Content.IpName + eniID := resp.Content.ElasticNetworkInterfaceId + + re := make([]IPSet, 0) + err = retry.OnError(wait.Backoff{ + Duration: 2 * time.Second, + Factor: 1, + Jitter: 0, + Steps: 3, + }, func(err error) bool { + return true + }, func() error { + content, err := a.ListLeniPrivateIPAddresses(ctx, eniID, ipName, "") + if err != nil { + return err + } + for _, data := range content.Data { + if data.IpName != ipName { + continue + } + if data.Status != "Available" { + return fmt.Errorf("ip %s status %s", data.PrivateIpAddress, data.Status) + } + + re = append(re, IPSet{ + Primary: false, + IPAddress: data.PrivateIpAddress, + IPName: data.IpName, + IPStatus: data.Status, + }) + } + return nil + }) + if err != nil { + re = append(re, IPSet{ + Primary: false, + IPName: ipName, + IPStatus: "", + }) + } + + return re, err +} + +func (a *OpenAPI) UnAssignLeniPrivateIPAddresses2(ctx context.Context, eniID string, ips []IPSet) error { + for _, ip := range ips { + if ip.IPName == "" { + continue + } + err := a.UnassignLeniPrivateIPAddress(ctx, eniID, ip.IPName) + if err != nil { + return err + } + } + return nil +} + +// WaitForLeniNetworkInterface wait status of eni +func (a *OpenAPI) WaitForLeniNetworkInterface(ctx context.Context, eniID string, status string, backoff wait.Backoff, ignoreNotExist bool) (*NetworkInterface, error) { + ctx, span := a.Tracer.Start(ctx, "WaitForNetworkInterface") + defer span.End() + + var eniInfo *NetworkInterface + if eniID == "" { + return nil, fmt.Errorf("eniID not set") + } + err := wait.ExponentialBackoff(backoff, + func() (done bool, err error) { + eni, err := a.DescribeLeniNetworkInterface(ctx, &DescribeNetworkInterfaceOptions{ + NetworkInterfaceIDs: &[]string{eniID}, + Status: &status, + }) + if err != nil { + return false, nil + } + if len(eni) == 0 && ignoreNotExist { + return true, apiErr.ErrNotFound + } + if len(eni) == 1 { + if string(status) != "" && status != eni[0].Status { + return false, nil + } + + eniInfo = eni[0] + return true, nil + } + return false, nil + }, + ) + if err != nil { + return nil, fmt.Errorf("error wait for eni %v to status %s, %w", eniID, status, err) + } + return eniInfo, nil +} diff --git a/pkg/aliyun/client/eni.go b/pkg/aliyun/client/eni.go new file mode 100644 index 00000000..412962b0 --- /dev/null +++ b/pkg/aliyun/client/eni.go @@ -0,0 +1,110 @@ +package client + +import ( + "context" + "errors" + + "k8s.io/apimachinery/pkg/util/wait" +) + +var ErrNotImplemented = errors.New("not implemented") + +func (a *OpenAPI) CreateNetworkInterfaceV2(ctx context.Context, opts ...CreateNetworkInterfaceOption) (*NetworkInterface, error) { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.CreateNetworkInterface(ctx, opts...) + case BackendAPIEFLO: + return a.CreateElasticNetworkInterfaceV2(ctx, opts...) + } + return nil, ErrNotImplemented +} + +func (a *OpenAPI) DescribeNetworkInterfaceV2(ctx context.Context, opts ...DescribeNetworkInterfaceOption) ([]*NetworkInterface, error) { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.DescribeNetworkInterface2(ctx, opts...) + case BackendAPIEFLO: + return a.DescribeLeniNetworkInterface(ctx, opts...) + } + return nil, ErrNotImplemented +} + +func (a *OpenAPI) AttachNetworkInterfaceV2(ctx context.Context, eniID, instanceID, trunkENIID string) error { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.AttachNetworkInterface(ctx, eniID, instanceID, trunkENIID) + case BackendAPIEFLO: + return nil + } + return ErrNotImplemented +} + +func (a *OpenAPI) DetachNetworkInterfaceV2(ctx context.Context, eniID, instanceID, trunkENIID string) error { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.DetachNetworkInterface(ctx, eniID, instanceID, trunkENIID) + case BackendAPIEFLO: + return nil + } + return ErrNotImplemented +} + +func (a *OpenAPI) DeleteNetworkInterfaceV2(ctx context.Context, eniID string) error { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.DeleteNetworkInterface(ctx, eniID) + case BackendAPIEFLO: + return a.DeleteElasticNetworkInterface(ctx, eniID) + } + return ErrNotImplemented +} + +func (a *OpenAPI) AssignPrivateIPAddressV2(ctx context.Context, opts ...AssignPrivateIPAddressOption) ([]IPSet, error) { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.AssignPrivateIPAddress2(ctx, opts...) + case BackendAPIEFLO: + return a.AssignLeniPrivateIPAddress2(ctx, opts...) + } + return nil, ErrNotImplemented +} + +func (a *OpenAPI) UnAssignPrivateIPAddressesV2(ctx context.Context, eniID string, ips []IPSet) error { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.UnAssignPrivateIPAddresses2(ctx, eniID, ips) + case BackendAPIEFLO: + return a.UnAssignLeniPrivateIPAddresses2(ctx, eniID, ips) + } + return ErrNotImplemented +} + +func (a *OpenAPI) AssignIpv6AddressesV2(ctx context.Context, opts ...AssignIPv6AddressesOption) ([]IPSet, error) { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.AssignIpv6Addresses2(ctx, opts...) + case BackendAPIEFLO: + return nil, ErrNotImplemented + } + return nil, ErrNotImplemented +} + +func (a *OpenAPI) UnAssignIpv6AddressesV2(ctx context.Context, eniID string, ips []IPSet) error { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.UnAssignIpv6Addresses2(ctx, eniID, ips) + case BackendAPIEFLO: + return ErrNotImplemented + } + return ErrNotImplemented +} + +func (a *OpenAPI) WaitForNetworkInterfaceV2(ctx context.Context, eniID string, status string, backoff wait.Backoff, ignoreNotExist bool) (*NetworkInterface, error) { + switch GetBackendAPI(ctx) { + case BackendAPIECS: + return a.WaitForNetworkInterface(ctx, eniID, status, backoff, ignoreNotExist) + case BackendAPIEFLO: + return a.WaitForLeniNetworkInterface(ctx, eniID, status, backoff, ignoreNotExist) + } + return nil, ErrNotImplemented +} diff --git a/pkg/aliyun/client/evc_2.go b/pkg/aliyun/client/evc_2.go new file mode 100644 index 00000000..264bc86e --- /dev/null +++ b/pkg/aliyun/client/evc_2.go @@ -0,0 +1,268 @@ +package client + +import ( + "context" + "fmt" + "net/netip" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/util/wait" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" + "github.com/AliyunContainerService/terway/pkg/ip" + "github.com/AliyunContainerService/terway/pkg/metric" +) + +// DescribeNetworkInterface2 list eni +func (a *OpenAPI) DescribeNetworkInterface2(ctx context.Context, opts ...DescribeNetworkInterfaceOption) ([]*NetworkInterface, error) { + ctx, span := a.Tracer.Start(ctx, APIDescribeNetworkInterfaces) + defer span.End() + + var result []*NetworkInterface + nextToken := "" + + for { + err := a.RateLimiter.Wait(ctx, APIDescribeNetworkInterfaces) + if err != nil { + return nil, err + } + + options := &DescribeNetworkInterfaceOptions{} + for _, opt := range opts { + opt.ApplyTo(options) + } + req := options.ECS() + req.NextToken = nextToken + req.MaxResults = requests.NewInteger(maxSinglePageSize) + + l := LogFields(logf.FromContext(ctx), req) + + start := time.Now() + resp, err := a.ClientSet.ECS().DescribeNetworkInterfaces(req) + metric.OpenAPILatency.WithLabelValues(APIDescribeNetworkInterfaces, fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) + if err != nil { + err = apiErr.WarpError(err) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "error describe eni") + return nil, err + } + for _, r := range resp.NetworkInterfaceSets.NetworkInterfaceSet { + result = append(result, FromDescribeResp(&r)) + } + + l.WithValues(LogFieldRequestID, resp.RequestId).Info("describe enis") + + if len(resp.NetworkInterfaceSets.NetworkInterfaceSet) < maxSinglePageSize { + break + } + if resp.NextToken == "" { + break + } + nextToken = resp.NextToken + } + return result, nil +} + +func (a *OpenAPI) AssignPrivateIPAddress2(ctx context.Context, opts ...AssignPrivateIPAddressOption) ([]IPSet, error) { + ctx, span := a.Tracer.Start(ctx, APIAssignPrivateIPAddress) + defer span.End() + + option := &AssignPrivateIPAddressOptions{} + for _, opt := range opts { + opt.ApplyAssignPrivateIPAddress(option) + } + + req, rollBackFunc, err := option.Finish(a.IdempotentKeyGen) + if err != nil { + return nil, err + } + l := LogFields(logf.FromContext(ctx), req) + + var ( + resp *ecs.AssignPrivateIpAddressesResponse + innerErr error + ) + + err = wait.ExponentialBackoffWithContext(ctx, *option.Backoff, func(ctx context.Context) (bool, error) { + innerErr = a.RateLimiter.Wait(ctx, APIAssignPrivateIPAddress) + if innerErr != nil { + return true, innerErr + } + start := time.Now() + resp, innerErr = a.ClientSet.ECS().AssignPrivateIpAddresses(req) + metric.OpenAPILatency.WithLabelValues(APIAssignPrivateIPAddress, fmt.Sprint(innerErr != nil)).Observe(metric.MsSince(start)) + if innerErr != nil { + innerErr = apiErr.WarpError(innerErr) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(innerErr)).Error(innerErr, "failed") + + if apiErr.ErrorIs(innerErr, apiErr.IsURLError, apiErr.WarpFn(apiErr.ErrThrottling, apiErr.ErrInternalError, apiErr.ErrOperationConflict)) { + return false, nil + } + + return true, innerErr + } + + return true, nil + }) + if err != nil { + rollBackFunc() + return nil, err + } + + ips, err := ip.ToIPAddrs(resp.AssignedPrivateIpAddressesSet.PrivateIpSet.PrivateIpAddress) + l.WithValues(LogFieldRequestID, resp.RequestId).Info("assign private ip", "ips", ips) + + return lo.Map(ips, func(item netip.Addr, _ int) IPSet { + return IPSet{ + IPAddress: item.String(), + IPStatus: "", + Primary: false, + } + }), err +} + +func (a *OpenAPI) UnAssignPrivateIPAddresses2(ctx context.Context, eniID string, ips []IPSet) error { + if len(ips) == 0 { + return nil + } + + ctx, span := a.Tracer.Start(ctx, APIUnAssignPrivateIPAddresses) + defer span.End() + + err := a.RateLimiter.Wait(ctx, APIUnAssignPrivateIPAddresses) + if err != nil { + return err + } + + req := ecs.CreateUnassignPrivateIpAddressesRequest() + req.NetworkInterfaceId = eniID + + str := lo.Map(ips, func(item IPSet, _ int) string { + return item.IPAddress + }) + req.PrivateIpAddress = &str + + l := LogFields(logf.FromContext(ctx), req) + + start := time.Now() + resp, err := a.ClientSet.ECS().UnassignPrivateIpAddresses(req) + metric.OpenAPILatency.WithLabelValues(APIUnAssignPrivateIPAddresses, fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) + + if err != nil { + err = apiErr.WarpError(err) + if apiErr.ErrorCodeIs(err, apiErr.ErrInvalidIPIPUnassigned, apiErr.ErrInvalidENINotFound) { + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Info("success") + return nil + } + + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "unassign private ip failed") + return err + } + l.WithValues(LogFieldRequestID, resp.RequestId).Info("success") + return nil +} + +// AssignIpv6Addresses2 assign ipv6 address +func (a *OpenAPI) AssignIpv6Addresses2(ctx context.Context, opts ...AssignIPv6AddressesOption) ([]IPSet, error) { + ctx, span := a.Tracer.Start(ctx, APIAssignIPv6Addresses) + defer span.End() + + option := &AssignIPv6AddressesOptions{} + for _, opt := range opts { + opt.ApplyAssignIPv6Addresses(option) + } + + req, rollBackFunc, err := option.Finish(a.IdempotentKeyGen) + if err != nil { + return nil, err + } + l := LogFields(logf.FromContext(ctx), req) + + var ( + resp *ecs.AssignIpv6AddressesResponse + innerErr error + ) + + err = wait.ExponentialBackoffWithContext(ctx, *option.Backoff, func(ctx context.Context) (bool, error) { + innerErr = a.RateLimiter.Wait(ctx, APIAssignIPv6Addresses) + if err != nil { + return true, innerErr + } + + start := time.Now() + resp, innerErr = a.ClientSet.ECS().AssignIpv6Addresses(req) + metric.OpenAPILatency.WithLabelValues(APIAssignIPv6Addresses, fmt.Sprint(innerErr != nil)).Observe(metric.MsSince(start)) + if innerErr != nil { + innerErr = apiErr.WarpError(innerErr) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(innerErr)).Error(innerErr, "failed") + + if apiErr.ErrorIs(innerErr, apiErr.IsURLError, apiErr.WarpFn(apiErr.ErrThrottling, apiErr.ErrInternalError, apiErr.ErrOperationConflict)) { + return false, nil + } + + return true, innerErr + } + + return true, nil + }) + if err != nil { + rollBackFunc() + return nil, err + } + + ips := lo.Map(resp.Ipv6Sets.Ipv6Address, func(item string, _ int) IPSet { + return IPSet{ + IPAddress: item, + IPStatus: "", + Primary: false, + } + }) + l.WithValues(LogFieldRequestID, resp.RequestId).Info("assign ipv6", "ips", ips) + + return ips, nil +} + +// UnAssignIpv6Addresses2 remove ip from eni +// return ok if 1. eni is released 2. ip is already released 3. release success +func (a *OpenAPI) UnAssignIpv6Addresses2(ctx context.Context, eniID string, ips []IPSet) error { + ctx, span := a.Tracer.Start(ctx, APIUnAssignIpv6Addresses) + defer span.End() + + if len(ips) == 0 { + return nil + } + + err := a.RateLimiter.Wait(ctx, APIUnAssignIpv6Addresses) + if err != nil { + return err + } + + req := ecs.CreateUnassignIpv6AddressesRequest() + req.NetworkInterfaceId = eniID + str := lo.Map(ips, func(item IPSet, _ int) string { + return item.IPAddress + }) + req.Ipv6Address = &str + + l := LogFields(logf.FromContext(ctx), req) + + start := time.Now() + resp, err := a.ClientSet.ECS().UnassignIpv6Addresses(req) + metric.OpenAPILatency.WithLabelValues(APIUnAssignIpv6Addresses, fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) + + if err != nil { + err = apiErr.WarpError(err) + if apiErr.ErrorCodeIs(err, apiErr.ErrInvalidIPIPUnassigned, apiErr.ErrInvalidENINotFound) { + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Info("success") + return nil + } + + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "unassign ipv6 ip failed") + return err + } + l.WithValues(LogFieldRequestID, resp.RequestId).Info("success") + return nil +} diff --git a/pkg/aliyun/client/interface_default.go b/pkg/aliyun/client/interface_default.go index 66887524..787f202d 100644 --- a/pkg/aliyun/client/interface_default.go +++ b/pkg/aliyun/client/interface_default.go @@ -50,8 +50,8 @@ func SetBackendAPI(ctx context.Context, b BackendAPI) context.Context { type ENI interface { CreateNetworkInterfaceV2(ctx context.Context, opts ...CreateNetworkInterfaceOption) (*NetworkInterface, error) DescribeNetworkInterfaceV2(ctx context.Context, opts ...DescribeNetworkInterfaceOption) ([]*NetworkInterface, error) - AttachNetworkInterfaceV2(ctx context.Context, opts ...AttachNetworkInterfaceOption) error - DetachNetworkInterfaceV2(ctx context.Context, opts ...DetachNetworkInterfaceOption) error + AttachNetworkInterface(ctx context.Context, eniID, instanceID, trunkENIID string) error + DetachNetworkInterface(ctx context.Context, eniID, instanceID, trunkENIID string) error DeleteNetworkInterfaceV2(ctx context.Context, eniID string) error AssignPrivateIPAddressV2(ctx context.Context, opts ...AssignPrivateIPAddressOption) ([]IPSet, error) UnAssignPrivateIPAddressesV2(ctx context.Context, eniID string, ips []IPSet) error diff --git a/pkg/aliyun/client/ecs_options.go b/pkg/aliyun/client/options.go similarity index 82% rename from pkg/aliyun/client/ecs_options.go rename to pkg/aliyun/client/options.go index 5608bbb8..d6f5775c 100644 --- a/pkg/aliyun/client/ecs_options.go +++ b/pkg/aliyun/client/options.go @@ -22,6 +22,8 @@ type NetworkInterfaceOptions struct { Status string NetworkInterfaceID string DeleteENIOnECSRelease *bool + + ZoneID string } type CreateNetworkInterfaceOption interface { @@ -84,6 +86,10 @@ func (c *CreateNetworkInterfaceOptions) ApplyCreateNetworkInterface(options *Cre if c.NetworkInterfaceOptions.DeleteENIOnECSRelease != nil { options.NetworkInterfaceOptions.DeleteENIOnECSRelease = c.NetworkInterfaceOptions.DeleteENIOnECSRelease } + + if c.NetworkInterfaceOptions.ZoneID != "" { + options.NetworkInterfaceOptions.ZoneID = c.NetworkInterfaceOptions.ZoneID + } } } @@ -140,6 +146,53 @@ func (c *CreateNetworkInterfaceOptions) Finish(idempotentKeyGen IdempotentKeyGen }, nil } +func (c *CreateNetworkInterfaceOptions) EFLO(idempotentKeyGen IdempotentKeyGen) (*eflo.CreateElasticNetworkInterfaceRequest, func(), error) { + if c.NetworkInterfaceOptions == nil { + return nil, nil, ErrInvalidArgs + } + + if c.NetworkInterfaceOptions.IPCount > 1 { + // eflo does not support multi ip in create + return nil, nil, ErrInvalidArgs + } + + if c.NetworkInterfaceOptions.VSwitchID == "" { + return nil, nil, ErrInvalidArgs + } + + req := eflo.CreateCreateElasticNetworkInterfaceRequest() + if c.NetworkInterfaceOptions.VSwitchID != "" { + req.VSwitchId = c.NetworkInterfaceOptions.VSwitchID + } + if len(c.NetworkInterfaceOptions.SecurityGroupIDs) > 0 { + req.SecurityGroupId = c.NetworkInterfaceOptions.SecurityGroupIDs[0] + } + req.Description = eniDescription + if c.NetworkInterfaceOptions.InstanceID != "" { + req.NodeId = c.NetworkInterfaceOptions.InstanceID + } + if c.NetworkInterfaceOptions.ZoneID != "" { + req.ZoneId = c.NetworkInterfaceOptions.ZoneID + } + + if req.SecurityGroupId == "" { + return nil, nil, ErrInvalidArgs + } + + argsHash := md5Hash(req) + req.ClientToken = idempotentKeyGen.GenerateKey(argsHash) + + if c.Backoff == nil { + c.Backoff = &wait.Backoff{ + Steps: 1, + } + } + + return req, func() { + idempotentKeyGen.PutBack(argsHash, req.ClientToken) + }, nil +} + type AssignPrivateIPAddressOption interface { ApplyAssignPrivateIPAddress(*AssignPrivateIPAddressOptions) } @@ -181,6 +234,29 @@ func (c *AssignPrivateIPAddressOptions) Finish(idempotentKeyGen IdempotentKeyGen }, nil } +func (c *AssignPrivateIPAddressOptions) EFLO(idempotentKeyGen IdempotentKeyGen) (*eflo.AssignLeniPrivateIpAddressRequest, func(), error) { + if c.NetworkInterfaceOptions == nil || + c.NetworkInterfaceOptions.NetworkInterfaceID == "" || + c.NetworkInterfaceOptions.IPCount != 1 { + return nil, nil, ErrInvalidArgs + } + req := eflo.CreateAssignLeniPrivateIpAddressRequest() + req.ElasticNetworkInterfaceId = c.NetworkInterfaceOptions.NetworkInterfaceID + + argsHash := md5Hash(req) + req.ClientToken = idempotentKeyGen.GenerateKey(argsHash) + + if c.Backoff == nil { + c.Backoff = &wait.Backoff{ + Steps: 1, + } + } + + return req, func() { + idempotentKeyGen.PutBack(argsHash, req.ClientToken) + }, nil +} + type AssignIPv6AddressesOption interface { ApplyAssignIPv6Addresses(*AssignIPv6AddressesOptions) } @@ -325,73 +401,3 @@ func (o *DescribeNetworkInterfaceOptions) EFLO() *eflo.ListElasticNetworkInterfa return req } - -type AttachNetworkInterfaceOption interface { - ApplyTo(*AttachNetworkInterfaceOptions) -} - -type AttachNetworkInterfaceOptions struct { - NetworkInterfaceID, InstanceID, TrunkNetworkInstanceId *string -} - -func (o *AttachNetworkInterfaceOptions) ApplyTo(in *AttachNetworkInterfaceOptions) { - if o.NetworkInterfaceID != nil { - in.NetworkInterfaceID = o.NetworkInterfaceID - } - if o.InstanceID != nil { - in.InstanceID = o.InstanceID - } - if o.TrunkNetworkInstanceId != nil { - in.TrunkNetworkInstanceId = o.TrunkNetworkInstanceId - } -} - -func (o *AttachNetworkInterfaceOptions) ECS() (*ecs.AttachNetworkInterfaceRequest, error) { - if o.NetworkInterfaceID == nil || o.InstanceID == nil { - return nil, ErrInvalidArgs - } - req := ecs.CreateAttachNetworkInterfaceRequest() - req.NetworkInterfaceId = *o.NetworkInterfaceID - req.InstanceId = *o.InstanceID - - if o.TrunkNetworkInstanceId != nil { - req.TrunkNetworkInstanceId = *o.TrunkNetworkInstanceId - } - - return req, nil -} - -type DetachNetworkInterfaceOption interface { - ApplyTo(*DetachNetworkInterfaceOptions) -} - -type DetachNetworkInterfaceOptions struct { - NetworkInterfaceID, InstanceID, TrunkENIID *string -} - -func (o *DetachNetworkInterfaceOptions) ApplyTo(in *DetachNetworkInterfaceOptions) { - if o.NetworkInterfaceID != nil { - in.NetworkInterfaceID = o.NetworkInterfaceID - } - if o.InstanceID != nil { - in.InstanceID = o.InstanceID - } - if o.TrunkENIID != nil { - in.TrunkENIID = o.TrunkENIID - } -} - -func (o *DetachNetworkInterfaceOptions) ECS() (*ecs.DetachNetworkInterfaceRequest, error) { - if o.NetworkInterfaceID == nil || o.InstanceID == nil { - return nil, ErrInvalidArgs - } - req := ecs.CreateDetachNetworkInterfaceRequest() - req.NetworkInterfaceId = *o.NetworkInterfaceID - req.InstanceId = *o.InstanceID - - if o.TrunkENIID != nil { - req.TrunkNetworkInstanceId = *o.TrunkENIID - } - - return req, nil -} diff --git a/pkg/aliyun/client/ecs_options_test.go b/pkg/aliyun/client/options_test.go similarity index 100% rename from pkg/aliyun/client/ecs_options_test.go rename to pkg/aliyun/client/options_test.go diff --git a/pkg/aliyun/client/types.go b/pkg/aliyun/client/types.go index 6d6904a5..3511558f 100644 --- a/pkg/aliyun/client/types.go +++ b/pkg/aliyun/client/types.go @@ -42,7 +42,8 @@ const ( ) const ( - LENIStatusAvailable string = "Available" + LENIStatusAvailable string = "Available" + LENIStatusUnattached string = "Unattached" ) const ( diff --git a/pkg/controller/node/predict.go b/pkg/controller/node/predict.go index ce3499ed..27d397c7 100644 --- a/pkg/controller/node/predict.go +++ b/pkg/controller/node/predict.go @@ -68,7 +68,7 @@ func predicateNode(o client.Object) bool { } func isECSNode(node *corev1.Node) bool { - if utils.ISLinJunNode(node) { + if utils.ISLinJunNode(node.Labels) { return false } if utils.ISVKNode(node) { diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index e117b185..e1d1090c 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -47,8 +47,8 @@ func ISVKNode(n *corev1.Node) bool { return n.Labels["type"] == "virtual-kubelet" } -func ISLinJunNode(n *corev1.Node) bool { - return n.Labels["alibabacloud.com/lingjun-worker"] == "true" +func ISLinJunNode(lb map[string]string) bool { + return lb["alibabacloud.com/lingjun-worker"] == "true" } // PodSandboxExited pod sandbox is exited From 82c892f1e606c6f7dc9b85a5bf0f62e5ec70d63e Mon Sep 17 00:00:00 2001 From: l1b0k Date: Mon, 17 Feb 2025 16:13:43 +0800 Subject: [PATCH 6/8] remove unused functions Signed-off-by: l1b0k --- pkg/aliyun/eni/eni.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/pkg/aliyun/eni/eni.go b/pkg/aliyun/eni/eni.go index 33443b9e..29d166ac 100644 --- a/pkg/aliyun/eni/eni.go +++ b/pkg/aliyun/eni/eni.go @@ -13,13 +13,10 @@ import ( // ENIInfoGetter interface to get eni information type ENIInfoGetter interface { - GetENIByMac(mac string) (*daemon.ENI, error) - GetENIPrivateAddressesByMACv2(mac string) ([]netip.Addr, error) GetENIPrivateIPv6AddressesByMACv2(mac string) ([]netip.Addr, error) GetENIs(containsMainENI bool) ([]*daemon.ENI, error) - GetSecondaryENIMACs() ([]string, error) } type ENIMetadata struct { @@ -126,22 +123,3 @@ func (e *ENIMetadata) GetENIs(containsMainENI bool) ([]*daemon.ENI, error) { } return enis, nil } - -// GetSecondaryENIMACs return secondary ENI macs -func (e *ENIMetadata) GetSecondaryENIMACs() ([]string, error) { - var result []string - - mainENIMac := instance.GetInstanceMeta().PrimaryMAC - - macs, err := metadata.GetENIsMAC() - if err != nil { - return nil, err - } - for _, mac := range macs { - if mac == mainENIMac { - continue - } - result = append(result, mac) - } - return result, nil -} From 3c2e2feb29ec21903dd4cfd2ed3eea1c62164be2 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Mon, 17 Feb 2025 16:56:31 +0800 Subject: [PATCH 7/8] controller use v2 api Signed-off-by: l1b0k --- pkg/controller/multi-ip/node/pool.go | 173 ++++++++++++++++++--------- pkg/controller/register.go | 1 + pkg/eni/node_reconcile.go | 110 ++++++++++++++++- 3 files changed, 227 insertions(+), 57 deletions(-) diff --git a/pkg/controller/multi-ip/node/pool.go b/pkg/controller/multi-ip/node/pool.go index 0fae91af..7fe4e3c1 100644 --- a/pkg/controller/multi-ip/node/pool.go +++ b/pkg/controller/multi-ip/node/pool.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "net/netip" "reflect" "sync" "sync/atomic" @@ -53,7 +52,8 @@ const ( finalizer = "network.alibabacloud.com/node-controller" - batchSize = 10 + ecsBatchSize = 10 + efloBatchSize = 1 ) var EventCh = make(chan event.GenericEvent, 1000) @@ -211,9 +211,20 @@ func (n *ReconcileNode) Reconcile(ctx context.Context, request reconcile.Request } // check if daemon has ready - if node.Spec.ENISpec == nil || - node.Spec.Pool == nil { - return reconcile.Result{}, nil + + if utils.ISLinJunNode(node.Labels) { + ctx = aliyunClient.SetBackendAPI(ctx, aliyunClient.BackendAPIEFLO) + if node.Spec.ENISpec == nil || + node.Spec.Pool == nil || + node.Spec.NodeCap.Adapters == 0 { + return reconcile.Result{}, nil + } + } else { + ctx = aliyunClient.SetBackendAPI(ctx, aliyunClient.BackendAPIECS) + if node.Spec.ENISpec == nil || + node.Spec.Pool == nil { + return reconcile.Result{}, nil + } } if types.NodeExclusiveENIMode(node.Labels) == types.ExclusiveENIOnly { @@ -313,8 +324,16 @@ func (n *ReconcileNode) syncWithAPI(ctx context.Context, node *networkv1beta1.No l := logf.FromContext(ctx).WithName("syncWithAPI") SyncOpenAPITotal.WithLabelValues(node.Name).Inc() + var err error + var enis []*aliyunClient.NetworkInterface // all eni attached to this instance is take into count. (exclude member eni) - enis, err := n.aliyun.DescribeNetworkInterface(ctx, "", nil, node.Spec.NodeMetadata.InstanceID, "", "", node.Spec.ENISpec.TagFilter) + opts := &aliyunClient.DescribeNetworkInterfaceOptions{ + InstanceID: &node.Spec.NodeMetadata.InstanceID, + } + if node.Spec.ENISpec.TagFilter != nil { + opts.Tags = &node.Spec.ENISpec.TagFilter + } + enis, err = n.aliyun.DescribeNetworkInterfaceV2(ctx, opts) if err != nil { return err } @@ -368,18 +387,26 @@ func (n *ReconcileNode) syncWithAPI(ctx context.Context, node *networkv1beta1.No // as the eni is not attached, so just delete it if node.Status.NetworkInterfaces[id].NetworkInterfaceType == networkv1beta1.ENITypeSecondary { var remote []*aliyunClient.NetworkInterface - remote, err = n.aliyun.DescribeNetworkInterface(ctx, "", []string{id}, "", "", "", node.Spec.ENISpec.TagFilter) + + opts = &aliyunClient.DescribeNetworkInterfaceOptions{ + InstanceID: &node.Spec.NodeMetadata.InstanceID, + NetworkInterfaceIDs: &[]string{id}, + } + if node.Spec.ENISpec.TagFilter != nil { + opts.Tags = &node.Spec.ENISpec.TagFilter + } + remote, err = n.aliyun.DescribeNetworkInterfaceV2(ctx, opts) + if err != nil { l.Error(err, "error get eni", "eni", id) continue } - // ignore eni , either be attached to other instance or be deleted or ignored by tag filter if len(remote) > 0 { switch remote[0].Status { case aliyunClient.ENIStatusAvailable: l.Info("delete eni not found in remote, but in cr", "eni", id) - err = n.aliyun.DeleteNetworkInterface(ctx, id) + err = n.aliyun.DeleteNetworkInterfaceV2(ctx, id) case aliyunClient.ENIStatusInUse: // ignore eni used by other instance default: @@ -685,10 +712,10 @@ func (n *ReconcileNode) addIP(ctx context.Context, unSucceedPods map[string]*Pod options := getEniOptions(node) // handle trunk/secondary eni - assignEniWithOptions(node, len(normalPods)+node.Spec.Pool.MinPoolSize, options, func(option *eniOptions) bool { + assignEniWithOptions(ctx, node, len(normalPods)+node.Spec.Pool.MinPoolSize, options, func(option *eniOptions) bool { return n.validateENI(ctx, option, []eniTypeKey{secondaryKey, trunkKey}) }) - assignEniWithOptions(node, len(rdmaPods), options, func(option *eniOptions) bool { + assignEniWithOptions(ctx, node, len(rdmaPods), options, func(option *eniOptions) bool { return n.validateENI(ctx, option, []eniTypeKey{rdmaKey}) }) @@ -871,7 +898,7 @@ func (n *ReconcileNode) validateENI(ctx context.Context, option *eniOptions, eni // assignEniWithOptions determine how many ip should be added to this eni. // In dual stack, ip on eni is automatically balanced. -func assignEniWithOptions(node *networkv1beta1.Node, toAdd int, options []*eniOptions, filterFunc func(option *eniOptions) bool) { +func assignEniWithOptions(ctx context.Context, node *networkv1beta1.Node, toAdd int, options []*eniOptions, filterFunc func(option *eniOptions) bool) { eniSpec := node.Spec.ENISpec toAddIPv4, toAddIPv6 := 0, 0 @@ -896,7 +923,7 @@ func assignEniWithOptions(node *networkv1beta1.Node, toAdd int, options []*eniOp if toAddIPv4 > 0 { leftQuota := node.Spec.NodeCap.IPv4PerAdapter - len(option.eniRef.IPv4) if leftQuota > 0 { - option.addIPv4N = min(leftQuota, toAddIPv4, batchSize) + option.addIPv4N = min(leftQuota, toAddIPv4, batchSize(ctx)) toAddIPv4 -= option.addIPv4N } else { option.isFull = true @@ -911,7 +938,7 @@ func assignEniWithOptions(node *networkv1beta1.Node, toAdd int, options []*eniOp if toAddIPv6 > 0 { leftQuota := node.Spec.NodeCap.IPv6PerAdapter - len(option.eniRef.IPv6) if leftQuota > 0 { - option.addIPv6N = min(leftQuota, toAddIPv6, batchSize) + option.addIPv6N = min(leftQuota, toAddIPv6, batchSize(ctx)) toAddIPv6 -= option.addIPv6N } else { option.isFull = true @@ -932,11 +959,11 @@ func assignEniWithOptions(node *networkv1beta1.Node, toAdd int, options []*eniOp } if toAddIPv4 > 0 { - option.addIPv4N = min(node.Spec.NodeCap.IPv4PerAdapter, toAddIPv4, batchSize) + option.addIPv4N = min(node.Spec.NodeCap.IPv4PerAdapter, toAddIPv4, batchSize(ctx)) toAddIPv4 -= option.addIPv4N } if toAddIPv6 > 0 { - option.addIPv6N = min(node.Spec.NodeCap.IPv6PerAdapter, toAddIPv6, batchSize) + option.addIPv6N = min(node.Spec.NodeCap.IPv6PerAdapter, toAddIPv6, batchSize(ctx)) toAddIPv6 -= option.addIPv6N } } @@ -1016,21 +1043,23 @@ func (n *ReconcileNode) handleStatus(ctx context.Context, node *networkv1beta1.N switch eni.Status { case aliyunClient.ENIStatusDeleting, aliyunClient.ENIStatusDetaching: - err := n.aliyun.DetachNetworkInterface(ctx, eni.ID, node.Spec.NodeMetadata.InstanceID, "") - if err != nil { - log.Error(err, "run gc failed") - continue - } - _, err = n.aliyun.WaitForNetworkInterface(ctx, eni.ID, aliyunClient.ENIStatusAvailable, backoff.Backoff(backoff.WaitENIStatus), true) - if err != nil { - if !errors.Is(err, apiErr.ErrNotFound) { + if !isEFLO(ctx) { + err := n.aliyun.DetachNetworkInterface(ctx, eni.ID, node.Spec.NodeMetadata.InstanceID, "") + if err != nil { log.Error(err, "run gc failed") continue } + _, err = n.aliyun.WaitForNetworkInterface(ctx, eni.ID, aliyunClient.ENIStatusAvailable, backoff.Backoff(backoff.WaitENIStatus), true) + if err != nil { + if !errors.Is(err, apiErr.ErrNotFound) { + log.Error(err, "run gc failed") + continue + } + } } // wait eni detached - err = n.aliyun.DeleteNetworkInterface(ctx, eni.ID) + err := n.aliyun.DeleteNetworkInterfaceV2(ctx, eni.ID) if err != nil { log.Error(err, "run gc failed") continue @@ -1042,26 +1071,27 @@ func (n *ReconcileNode) handleStatus(ctx context.Context, node *networkv1beta1.N delete(node.Status.NetworkInterfaces, eni.ID) case aliyunClient.ENIStatusInUse: var waitTime time.Duration - ips := make([]netip.Addr, 0) + ips := make([]aliyunClient.IPSet, 0) for _, ip := range eni.IPv4 { if ip.Status == networkv1beta1.IPStatusDeleting { - addr, err := netip.ParseAddr(ip.IP) - if err == nil { - ips = append(ips, addr) - } + ips = append(ips, aliyunClient.IPSet{ + Primary: false, + IPAddress: ip.IP, + IPName: ip.IPName, + }) } } - ips = lo.Subset(ips, 0, batchSize) + ips = lo.Subset(ips, 0, uint(batchSize(ctx))) if len(ips) > 0 { - err := n.aliyun.UnAssignPrivateIPAddresses(ctx, eni.ID, ips) + err := n.aliyun.UnAssignPrivateIPAddressesV2(ctx, eni.ID, ips) if err != nil { continue } MetaCtx(ctx).StatusChanged.Store(true) - lo.ForEach(ips, func(ip netip.Addr, _ int) { - delete(eni.IPv4, ip.String()) + lo.ForEach(ips, func(ip aliyunClient.IPSet, _ int) { + delete(eni.IPv4, ip.IPAddress) }) waitTime = 1 * time.Second @@ -1070,26 +1100,27 @@ func (n *ReconcileNode) handleStatus(ctx context.Context, node *networkv1beta1.N ips = ips[:0] for _, ip := range eni.IPv6 { if ip.Status == networkv1beta1.IPStatusDeleting { - addr, err := netip.ParseAddr(ip.IP) - if err == nil { - ips = append(ips, addr) - } + ips = append(ips, aliyunClient.IPSet{ + Primary: false, + IPAddress: ip.IP, + IPName: ip.IPName, + }) } } - ips = lo.Subset(ips, 0, batchSize) + ips = lo.Subset(ips, 0, uint(batchSize(ctx))) if len(ips) > 0 { if waitTime > 0 { time.Sleep(waitTime) } - err := n.aliyun.UnAssignIpv6Addresses(ctx, eni.ID, ips) + err := n.aliyun.UnAssignIpv6AddressesV2(ctx, eni.ID, ips) if err != nil { continue } MetaCtx(ctx).StatusChanged.Store(true) - lo.ForEach(ips, func(ip netip.Addr, _ int) { - delete(eni.IPv6, ip.String()) + lo.ForEach(ips, func(ip aliyunClient.IPSet, _ int) { + delete(eni.IPv6, ip.IPAddress) }) } default: @@ -1167,10 +1198,11 @@ func (n *ReconcileNode) createENI(ctx context.Context, node *networkv1beta1.Node } bo := backoff.Backoff(backoff.ENICreate) - tags := node.Spec.ENISpec.Tag - if tags == nil { - tags = map[string]string{} + tags := make(map[string]string, len(node.Spec.ENISpec.Tag)) + for k, v := range node.Spec.ENISpec.Tag { + tags[k] = v } + // keep the ds behave tags[types.NetworkInterfaceTagCreatorKey] = types.NetworkInterfaceTagCreatorValue createOpts := &aliyunClient.CreateNetworkInterfaceOptions{ @@ -1181,11 +1213,15 @@ func (n *ReconcileNode) createENI(ctx context.Context, node *networkv1beta1.Node Tags: node.Spec.ENISpec.Tag, IPCount: opt.addIPv4N, IPv6Count: opt.addIPv6N, + + ZoneID: node.Spec.NodeMetadata.ZoneID, // eflo + InstanceID: node.Spec.NodeMetadata.InstanceID, }, + Backoff: &bo, } - result, err := n.aliyun.CreateNetworkInterface(ctx, typeOption, createOpts) + result, err := n.aliyun.CreateNetworkInterfaceV2(ctx, typeOption, createOpts) if err != nil { if apiErr.ErrorCodeIs(err, apiErr.InvalidVSwitchIDIPNotEnough, apiErr.QuotaExceededPrivateIPAddress) { // block @@ -1203,7 +1239,7 @@ func (n *ReconcileNode) createENI(ctx context.Context, node *networkv1beta1.Node rollbackCtx, rollbackCancel := context.WithTimeout(context.Background(), 60*time.Second) defer rollbackCancel() - innerErr := n.aliyun.DeleteNetworkInterface(rollbackCtx, result.NetworkInterfaceID) + innerErr := n.aliyun.DeleteNetworkInterfaceV2(rollbackCtx, result.NetworkInterfaceID) if innerErr == nil { return } @@ -1216,13 +1252,26 @@ func (n *ReconcileNode) createENI(ctx context.Context, node *networkv1beta1.Node MetaCtx(ctx).StatusChanged.Store(true) } }() - err = n.aliyun.AttachNetworkInterface(ctx, result.NetworkInterfaceID, node.Spec.NodeMetadata.InstanceID, "") - if err != nil { - return err + + var status string + if !isEFLO(ctx) { + err = n.aliyun.AttachNetworkInterface(ctx, result.NetworkInterfaceID, node.Spec.NodeMetadata.InstanceID, "") + if err != nil { + return err + } + if err != nil { + return err + } + + time.Sleep(3 * time.Second) + + status = aliyunClient.ENIStatusInUse + + } else { + status = aliyunClient.LENIStatusAvailable } - time.Sleep(3 * time.Second) - eni, err := n.aliyun.WaitForNetworkInterface(ctx, result.NetworkInterfaceID, aliyunClient.ENIStatusInUse, backoff.Backoff(backoff.WaitENIStatus), false) + eni, err := n.aliyun.WaitForNetworkInterfaceV2(ctx, result.NetworkInterfaceID, status, backoff.Backoff(backoff.WaitENIStatus), false) if err != nil { return err } @@ -1246,13 +1295,14 @@ func (n *ReconcileNode) assignIP(ctx context.Context, opt *eniOptions) error { // nb(l1b0k): ENi does not support assigning both IPv4 and IPv6 simultaneously. if opt.addIPv4N > 0 { bo := backoff.Backoff(backoff.ENIIPOps) - result, err := n.aliyun.AssignPrivateIPAddress(ctx, &aliyunClient.AssignPrivateIPAddressOptions{ + result, err := n.aliyun.AssignPrivateIPAddressV2(ctx, &aliyunClient.AssignPrivateIPAddressOptions{ NetworkInterfaceOptions: &aliyunClient.NetworkInterfaceOptions{ NetworkInterfaceID: opt.eniRef.ID, IPCount: opt.addIPv4N, }, Backoff: &bo, }) + if err != nil { if apiErr.ErrorCodeIs(err, apiErr.InvalidVSwitchIDIPNotEnough, apiErr.QuotaExceededPrivateIPAddress) { // block @@ -1267,7 +1317,7 @@ func (n *ReconcileNode) assignIP(ctx context.Context, opt *eniOptions) error { } for _, ip := range result { addIPToMap(opt.eniRef.IPv4, &networkv1beta1.IP{ - IP: ip.String(), + IP: ip.IPAddress, Status: networkv1beta1.IPStatusValid, }) } @@ -1275,7 +1325,7 @@ func (n *ReconcileNode) assignIP(ctx context.Context, opt *eniOptions) error { } if opt.addIPv6N > 0 { bo := backoff.Backoff(backoff.ENIIPOps) - result, err := n.aliyun.AssignIpv6Addresses(ctx, &aliyunClient.AssignIPv6AddressesOptions{ + result, err := n.aliyun.AssignIpv6AddressesV2(ctx, &aliyunClient.AssignIPv6AddressesOptions{ NetworkInterfaceOptions: &aliyunClient.NetworkInterfaceOptions{ NetworkInterfaceID: opt.eniRef.ID, IPv6Count: opt.addIPv6N, @@ -1296,7 +1346,7 @@ func (n *ReconcileNode) assignIP(ctx context.Context, opt *eniOptions) error { } for _, ip := range result { addIPToMap(opt.eniRef.IPv6, &networkv1beta1.IP{ - IP: ip.String(), + IP: ip.IPAddress, Status: networkv1beta1.IPStatusValid, }) } @@ -1446,3 +1496,14 @@ func isIPNotEnough(err error) bool { } return false } + +func isEFLO(ctx context.Context) bool { + return aliyunClient.GetBackendAPI(ctx) == aliyunClient.BackendAPIEFLO +} + +func batchSize(ctx context.Context) int { + if isEFLO(ctx) { + return efloBatchSize + } + return ecsBatchSize +} diff --git a/pkg/controller/register.go b/pkg/controller/register.go index 028ec4f6..94420e48 100644 --- a/pkg/controller/register.go +++ b/pkg/controller/register.go @@ -36,6 +36,7 @@ import ( type Interface interface { aliyunClient.VPC aliyunClient.ECS + aliyunClient.ENI } type ControllerCtx struct { diff --git a/pkg/eni/node_reconcile.go b/pkg/eni/node_reconcile.go index d7852fe2..0d3b2044 100644 --- a/pkg/eni/node_reconcile.go +++ b/pkg/eni/node_reconcile.go @@ -18,6 +18,7 @@ import ( "github.com/AliyunContainerService/terway/deviceplugin" "github.com/AliyunContainerService/terway/pkg/aliyun/instance" networkv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" + "github.com/AliyunContainerService/terway/pkg/utils" "github.com/AliyunContainerService/terway/pkg/utils/nodecap" "github.com/AliyunContainerService/terway/types" "github.com/AliyunContainerService/terway/types/daemon" @@ -57,6 +58,9 @@ func (r *nodeReconcile) Reconcile(ctx context.Context, request reconcile.Request } return reconcile.Result{}, err } + if utils.ISLinJunNode(k8sNode.Labels) { + return r.handleEFLO(ctx, k8sNode, node) + } eniConfig, err := daemon.ConfigFromConfigMap(ctx, r.client, node.Name) if err != nil { @@ -103,7 +107,12 @@ func (r *nodeReconcile) Reconcile(ctx context.Context, request reconcile.Request } if len(vswitchOptions) == 0 { // if user forget to set vsw , we still rely on metadata to get the actual one - vswitchOptions = append(vswitchOptions, instance.GetInstanceMeta().VSwitchID) + + switchID, err := instance.VSwitchID() + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to get vsw from metadata, %w", err) + } + vswitchOptions = append(vswitchOptions, switchID) } // below fields allows to change @@ -194,6 +203,105 @@ func (r *nodeReconcile) Reconcile(ctx context.Context, request reconcile.Request return reconcile.Result{}, nil } +func (r *nodeReconcile) handleEFLO(ctx context.Context, k8sNode *corev1.Node, node *networkv1beta1.Node) (reconcile.Result, error) { + l := log.FromContext(ctx) + + eniConfig, err := daemon.ConfigFromConfigMap(ctx, r.client, node.Name) + if err != nil { + r.record.Event(k8sNode, "Warning", "ConfigError", err.Error()) + return reconcile.Result{}, err + } + + beforeStatus, err := runtime.DefaultUnstructuredConverter.ToUnstructured(node.DeepCopy()) + if err != nil { + return reconcile.Result{}, err + } + + //node.Spec.ENISpec = nil + + node.Spec.ENISpec = &networkv1beta1.ENISpec{ + EnableIPv4: true, + EnableIPv6: false, + } + + if node.Labels == nil { + node.Labels = map[string]string{} + } + node.Labels[types.LinJunNodeLabel] = "true" + + regionID, err := instance.EFLORegionID() + if err != nil { + return reconcile.Result{}, err + } + instanceType, err := instance.EFLOInstanceType() + if err != nil { + return reconcile.Result{}, err + } + nodeID, err := instance.EFLONodeID() + if err != nil { + return reconcile.Result{}, err + } + zoneID, err := instance.EFLOZoneID() + if err != nil { + return reconcile.Result{}, err + } + node.Spec.NodeMetadata.RegionID = regionID + node.Spec.NodeMetadata.InstanceType = instanceType + node.Spec.NodeMetadata.InstanceID = nodeID + node.Spec.NodeMetadata.ZoneID = zoneID + + vswitchOptions := []string{} + for k, v := range eniConfig.VSwitches { + if k == node.Spec.NodeMetadata.ZoneID { + vswitchOptions = append(vswitchOptions, v...) + } + } + if len(vswitchOptions) == 0 { + return reconcile.Result{}, fmt.Errorf("failed to get vsw for zone %s, %w", zoneID, err) + } + + policy := networkv1beta1.VSwitchSelectionPolicyRandom + switch eniConfig.VSwitchSelectionPolicy { + case "ordered": + // keep the previous behave + policy = networkv1beta1.VSwitchSelectionPolicyMost + } + + node.Spec.ENISpec.VSwitchOptions = vswitchOptions + node.Spec.ENISpec.VSwitchSelectPolicy = policy + node.Spec.ENISpec.SecurityGroupIDs = eniConfig.GetSecurityGroups() + node.Spec.ENISpec.Tag = eniConfig.ENITags + node.Spec.ENISpec.TagFilter = eniConfig.ENITagFilter + node.Spec.ENISpec.ResourceGroupID = eniConfig.ResourceGroupID + + if node.Spec.NodeCap.Adapters > 0 { + node.Spec.Flavor = nil + node.Spec.Flavor = append(node.Spec.Flavor, networkv1beta1.Flavor{ + NetworkInterfaceType: networkv1beta1.ENITypeSecondary, + NetworkInterfaceTrafficMode: networkv1beta1.NetworkInterfaceTrafficModeStandard, + Count: node.Spec.NodeCap.Adapters - 1, + }) + } + + node.Spec.Pool = &networkv1beta1.PoolSpec{ + MaxPoolSize: eniConfig.MaxPoolSize, + MinPoolSize: eniConfig.MinPoolSize, + } + + afterStatus, err := runtime.DefaultUnstructuredConverter.ToUnstructured(node.DeepCopy()) + if err != nil { + return reconcile.Result{}, err + } + + if !reflect.DeepEqual(beforeStatus, afterStatus) { + err = r.client.Update(ctx, node) + l.Info("update node spec") + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *nodeReconcile) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). From 4dc0211d69a19dd7aa80af2c89eb3d88c7c5943c Mon Sep 17 00:00:00 2001 From: l1b0k Date: Mon, 17 Feb 2025 19:05:36 +0800 Subject: [PATCH 8/8] Refactor(instance): Refactor the way instance metadata is retrieved - Modify the logic for retrieving instance metadata, optimizing with interfaces and caching - Refactor the way EFLO instance metadata is retrieved - Update relevant unit tests Signed-off-by: l1b0k --- .github/workflows/build-policy.yml | 6 +- .github/workflows/build.yml | 4 +- .github/workflows/check.yml | 16 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/release.yml | 2 +- .golangci.yml | 4 - Makefile | 4 +- cmd/terway-cli/cni_linux.go | 56 +++- cmd/terway-cli/cni_linux_test.go | 41 +++ .../terway-controlplane.go | 12 + daemon/builder.go | 74 ++++- daemon/config.go | 23 +- daemon/config_test.go | 67 +++-- daemon/daemon.go | 4 +- daemon/daemon_test.go | 8 +- deploy/images/terway-controlplane/Dockerfile | 2 +- deploy/images/terway/Dockerfile | 2 +- examples/maxpods/maxpods.go | 16 +- go.mod | 84 +++--- go.sum | 158 +++++------ pkg/aliyun/client/{evc_2.go => ecs_2.go} | 0 pkg/aliyun/client/mocks/ECS.go | 2 +- pkg/aliyun/client/mocks/EFLO.go | 2 +- pkg/aliyun/client/mocks/LimitProvider.go | 32 ++- pkg/aliyun/client/mocks/VPC.go | 2 +- pkg/aliyun/credential/aliyun_client_mgr.go | 9 +- pkg/aliyun/eni/eni.go | 5 +- pkg/aliyun/instance/ecs.go | 96 +++++-- pkg/aliyun/instance/eflo.go | 104 +++++++ pkg/aliyun/instance/eflo_linux.go | 40 --- pkg/aliyun/instance/eflo_test.go | 34 +++ pkg/aliyun/instance/eflo_unlinux.go | 7 - pkg/aliyun/instance/instance.go | 47 +--- pkg/aliyun/instance/mocks/Interface.go | 192 +++++++++++++ pkg/controller/common/types_default.go | 6 + pkg/controller/common/types_default_test.go | 111 ++++++++ pkg/controller/mocks/Interface.go | 266 +++++++++++++++++- pkg/controller/multi-ip/node/pool.go | 4 +- pkg/controller/multi-ip/node/pool_test.go | 42 +-- pkg/controller/node/node.go | 48 +++- pkg/controller/node/node_controller_test.go | 64 +++-- pkg/controller/node/predict.go | 30 +- pkg/controller/node/predict_test.go | 137 ++++----- pkg/controller/register.go | 1 + pkg/eni/local.go | 2 +- pkg/eni/local_test.go | 48 ++-- pkg/eni/manager.go | 8 +- pkg/eni/manager_test.go | 10 +- pkg/eni/node_reconcile.go | 16 +- pkg/eni/node_reconcile_test.go | 107 +++++++ pkg/eni/suite_test.go | 106 +++++++ pkg/factory/aliyun/aliyun.go | 7 +- pkg/factory/aliyun/eflo.go | 2 +- pkg/factory/mocks/Factory.go | 2 +- pkg/feature/feature.go | 3 + pkg/k8s/mocks/Kubernetes.go | 16 +- pkg/utils/k8s.go | 3 +- .../nodecap/mocks/NodeCapabilitiesStore.go | 6 +- types/config.go | 68 ----- types/daemon/config.go | 64 +++++ types/k8s.go | 2 + 61 files changed, 1745 insertions(+), 591 deletions(-) create mode 100644 cmd/terway-cli/cni_linux_test.go rename pkg/aliyun/client/{evc_2.go => ecs_2.go} (100%) create mode 100644 pkg/aliyun/instance/eflo.go delete mode 100644 pkg/aliyun/instance/eflo_linux.go create mode 100644 pkg/aliyun/instance/eflo_test.go delete mode 100644 pkg/aliyun/instance/eflo_unlinux.go create mode 100644 pkg/aliyun/instance/mocks/Interface.go create mode 100644 pkg/controller/common/types_default_test.go create mode 100644 pkg/eni/node_reconcile_test.go create mode 100644 pkg/eni/suite_test.go delete mode 100644 types/config.go diff --git a/.github/workflows/build-policy.yml b/.github/workflows/build-policy.yml index b87ed5b1..6ca4fdcb 100644 --- a/.github/workflows/build-policy.yml +++ b/.github/workflows/build-policy.yml @@ -15,12 +15,14 @@ concurrency: jobs: build-policy: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - + with: + image: tonistiigi/binfmt:qemu-v7.0.0 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d5b5bf1..3382c2cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,12 +18,14 @@ concurrency: jobs: build-terway: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 + with: + image: tonistiigi/binfmt:qemu-v7.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 8bb5f2bc..46adc52f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -10,12 +10,12 @@ concurrency: jobs: go-test: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 1.23.2 + go-version: 1.24.0 - name: Test run: | go=$(which go) @@ -30,12 +30,12 @@ jobs: verbose: true go-mod: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 1.23.2 + go-version: 1.24.0 - name: Check module vendoring run: | go mod tidy @@ -44,21 +44,21 @@ jobs: go-lint: name: lint - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 1.23.2 + go-version: 1.24.0 cache: false - name: Run golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.61 + version: v1.64.5 args: --config=.golangci.yml super-linter: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Lint Code Base diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6029095b..7994e12b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: actions: read contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index af99c2ea..23ca1a44 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ concurrency: jobs: release: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Build Changelog diff --git a/.golangci.yml b/.golangci.yml index 2b0cf2b4..626d8669 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,7 +26,3 @@ linters: linters-settings: errcheck: check-blank: false - govet: - check-shadowing: false - maligned: - suggest-new: true \ No newline at end of file diff --git a/Makefile b/Makefile index 7064149f..aa847f44 100644 --- a/Makefile +++ b/Makefile @@ -114,9 +114,9 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) ## Tool Versions -CONTROLLER_TOOLS_VERSION ?= v0.14.0 +CONTROLLER_TOOLS_VERSION ?= v0.17.2 ENVTEST_VERSION ?= latest -GOLANGCI_LINT_VERSION ?= v1.61.0 +GOLANGCI_LINT_VERSION ?= v1.64.5 .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. diff --git a/cmd/terway-cli/cni_linux.go b/cmd/terway-cli/cni_linux.go index eb1e482f..1770de56 100644 --- a/cmd/terway-cli/cni_linux.go +++ b/cmd/terway-cli/cni_linux.go @@ -3,8 +3,10 @@ package main import ( "errors" "fmt" + "strconv" + "strings" + "syscall" - "github.com/docker/docker/pkg/parsers/kernel" "github.com/vishvananda/netlink" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -60,6 +62,54 @@ func allowEBPFNetworkPolicy(require bool) (bool, error) { return require, nil } -func checkKernelVersion(k, major, minor int) bool { - return kernel.CheckKernelVersion(k, major, minor) +func checkKernelVersion(iMajor, iMinor, iPatch int) bool { + var un syscall.Utsname + _ = syscall.Uname(&un) + var sb strings.Builder + for _, b := range un.Release[:] { + if b == 0 { + break + } + sb.WriteByte(byte(b)) + } + major, minor, patch, ok := parseRelease(sb.String()) + return ok && (major > iMajor || + major == iMajor && minor > iMinor || + major == iMajor && minor == iMinor && iPatch >= patch) +} + +// parseRelease parses a dot-separated version number. It follows the semver +// syntax, but allows the minor and patch versions to be elided. +// +// This is a copy of the Go runtime's parseRelease from +// https://golang.org/cl/209597. +func parseRelease(rel string) (major, minor, patch int, ok bool) { + // Strip anything after a dash or plus. + for i := 0; i < len(rel); i++ { + if rel[i] == '-' || rel[i] == '+' { + rel = rel[:i] + break + } + } + + next := func() (int, bool) { + for i := 0; i < len(rel); i++ { + if rel[i] == '.' { + ver, err := strconv.Atoi(rel[:i]) + rel = rel[i+1:] + return ver, err == nil + } + } + ver, err := strconv.Atoi(rel) + rel = "" + return ver, err == nil + } + if major, ok = next(); !ok || rel == "" { + return + } + if minor, ok = next(); !ok || rel == "" { + return + } + patch, ok = next() + return } diff --git a/cmd/terway-cli/cni_linux_test.go b/cmd/terway-cli/cni_linux_test.go new file mode 100644 index 00000000..6a3ad8da --- /dev/null +++ b/cmd/terway-cli/cni_linux_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_parseRelease(t *testing.T) { + type args struct { + rel string + } + tests := []struct { + name string + args args + wantMajor int + wantMinor int + wantPatch int + wantOk bool + }{ + { + name: "test1", + args: args{ + rel: "6.8.0-51-generic", + }, + wantMajor: 6, + wantMinor: 8, + wantPatch: 0, + wantOk: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotMajor, gotMinor, gotPatch, gotOk := parseRelease(tt.args.rel) + assert.Equalf(t, tt.wantMajor, gotMajor, "parseRelease(%v)", tt.args.rel) + assert.Equalf(t, tt.wantMinor, gotMinor, "parseRelease(%v)", tt.args.rel) + assert.Equalf(t, tt.wantPatch, gotPatch, "parseRelease(%v)", tt.args.rel) + assert.Equalf(t, tt.wantOk, gotOk, "parseRelease(%v)", tt.args.rel) + }) + } +} diff --git a/cmd/terway-controlplane/terway-controlplane.go b/cmd/terway-controlplane/terway-controlplane.go index 35443d2a..a10f0aa7 100644 --- a/cmd/terway-controlplane/terway-controlplane.go +++ b/cmd/terway-controlplane/terway-controlplane.go @@ -23,6 +23,7 @@ import ( "math/rand" "net" "os" + "strings" "time" "github.com/samber/lo" @@ -41,7 +42,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + cliflag "k8s.io/component-base/cli/flag" "k8s.io/klog/v2/textlogger" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -92,9 +95,12 @@ func main() { var ( configFilePath string credentialFilePath string + featureGates map[string]bool ) flag.StringVar(&configFilePath, "config", "/etc/config/ctrl-config.yaml", "config file for controlplane") flag.StringVar(&credentialFilePath, "credential", "/etc/credential/ctrl-secret.yaml", "secret file for controlplane") + flag.Var(cliflag.NewMapStringBool(&featureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ + "Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")) logCfg := textlogger.NewConfig() logCfg.AddFlags(flag.CommandLine) @@ -104,6 +110,12 @@ func main() { ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig())) log.Info(version.Version) + err := utilfeature.DefaultMutableFeatureGate.SetFromMap(featureGates) + if err != nil { + log.Error(err, "unable to set feature gates") + os.Exit(1) + } + ctx := ctrl.SetupSignalHandler() cfg, err := controlplane.ParseAndValidate(configFilePath, credentialFilePath) diff --git a/daemon/builder.go b/daemon/builder.go index 6305fc41..2e6305de 100644 --- a/daemon/builder.go +++ b/daemon/builder.go @@ -39,7 +39,8 @@ type NetworkServiceBuilder struct { limit *client.Limits - err error + eflo bool + err error } func NewNetworkServiceBuilder(ctx context.Context) *NetworkServiceBuilder { @@ -120,6 +121,10 @@ func (b *NetworkServiceBuilder) InitK8S() *NetworkServiceBuilder { b.service.daemonMode = daemon.ModeENIOnly } + if utils.ISLinJunNode(b.service.k8s.Node().Labels) { + b.eflo = true + instance.Init(&instance.EFLO{}) + } return b } @@ -152,10 +157,10 @@ func (b *NetworkServiceBuilder) LoadDynamicConfig() *NetworkServiceBuilder { } func (b *NetworkServiceBuilder) setupAliyunClient() error { - if os.Getenv("TERWAY_DEPLOY_ENV") == envEFLO { - instance.SetPopulateFunc(instance.EfloPopulate) + regionID, err := instance.GetInstanceMeta().GetRegionID() + if err != nil { + return err } - meta := instance.GetInstanceMeta() var providers []credential.Interface if string(b.config.AccessID) != "" && string(b.config.AccessSecret) != "" { @@ -164,7 +169,7 @@ func (b *NetworkServiceBuilder) setupAliyunClient() error { providers = append(providers, credential.NewEncryptedCredentialProvider(utils.NormalizePath(b.config.CredentialPath))) providers = append(providers, credential.NewMetadataProvider()) - clientSet, err := credential.NewClientMgr(meta.RegionID, providers...) + clientSet, err := credential.NewClientMgr(regionID, providers...) if err != nil { return err } @@ -184,14 +189,18 @@ func (b *NetworkServiceBuilder) initInstanceLimit() error { return fmt.Errorf("k8s node not found") } provider := client.LimitProviders["ecs"] - if os.Getenv("TERWAY_DEPLOY_ENV") == envEFLO { + if b.eflo { provider = client.LimitProviders["eflo"] limit, err := provider.GetLimitFromAnno(node.Annotations) if err != nil { return err } if limit == nil { - limit, err = provider.GetLimit(b.aliyunClient, instance.GetInstanceMeta().InstanceID) + instanceID, err := instance.GetInstanceMeta().GetInstanceID() + if err != nil { + return err + } + limit, err = provider.GetLimit(b.aliyunClient, instanceID) if err != nil { return fmt.Errorf("upable get instance limit, %w", err) } @@ -202,10 +211,28 @@ func (b *NetworkServiceBuilder) initInstanceLimit() error { if err != nil { return err } - if limit == nil || instance.GetInstanceMeta().InstanceType != limit.InstanceTypeID { - limit, err = provider.GetLimit(b.aliyunClient, instance.GetInstanceMeta().InstanceType) + + if limit != nil { + instanceType, err := instance.GetInstanceMeta().GetInstanceType() if err != nil { - return fmt.Errorf("upable get instance limit, %w", err) + return err + } + if limit.InstanceTypeID != instanceType { + limit = nil + } + } + + if limit == nil { + instanceType, err := instance.GetInstanceMeta().GetInstanceType() + if err != nil { + return err + } + + if instanceType != limit.InstanceTypeID { + limit, err = provider.GetLimit(b.aliyunClient, instanceType) + if err != nil { + return fmt.Errorf("upable get instance limit, %w", err) + } } } b.limit = limit @@ -227,6 +254,29 @@ func (b *NetworkServiceBuilder) setupENIManager() error { eniConfig.EnableIPv4 = enableIPv4 eniConfig.EnableIPv6 = enableIPv6 + zoneID, err := instance.GetInstanceMeta().GetZoneID() + if err != nil { + return err + } + instanceID, err := instance.GetInstanceMeta().GetInstanceID() + if err != nil { + return err + } + vswitchID, err := instance.GetInstanceMeta().GetVSwitchID() + if err != nil { + return err + } + + if eniConfig.ZoneID == "" { + eniConfig.ZoneID = zoneID + } + if eniConfig.InstanceID == "" { + eniConfig.InstanceID = instanceID + } + if len(eniConfig.VSwitchOptions) == 0 { + eniConfig.VSwitchOptions = []string{vswitchID} + } + // fall back to use primary eni's sg if len(eniConfig.SecurityGroupIDs) == 0 { enis, err := b.aliyunClient.DescribeNetworkInterface(b.ctx, "", nil, eniConfig.InstanceID, "Primary", "", nil) @@ -361,7 +411,7 @@ func (b *NetworkServiceBuilder) setupENIManager() error { eniList = append(eniList, eni.NewLocal(nil, "secondary", factory, poolConfig)) } - eniManager := eni.NewManager(poolConfig.MinPoolSize, poolConfig.MaxPoolSize, poolConfig.Capacity, 30*time.Second, eniList, types.EniSelectionPolicy(b.config.EniSelectionPolicy), b.service.k8s) + eniManager := eni.NewManager(poolConfig.MinPoolSize, poolConfig.MaxPoolSize, poolConfig.Capacity, 30*time.Second, eniList, daemon.EniSelectionPolicy(b.config.EniSelectionPolicy), b.service.k8s) b.service.eniMgr = eniManager err = eniManager.Run(b.ctx, &b.service.wg, podResources) if err != nil { @@ -405,7 +455,7 @@ func (b *NetworkServiceBuilder) PostInitForCRDV2() *NetworkServiceBuilder { return b } crdv2 := eni.NewCRDV2(b.service.k8s.NodeName(), b.namespace) - mgr := eni.NewManager(0, 0, 0, 0, []eni.NetworkInterface{crdv2}, types.EniSelectionPolicy(b.config.EniSelectionPolicy), nil) + mgr := eni.NewManager(0, 0, 0, 0, []eni.NetworkInterface{crdv2}, daemon.EniSelectionPolicy(b.config.EniSelectionPolicy), nil) svc := b.RunENIMgr(b.ctx, mgr) go b.service.startGarbageCollectionLoop(b.ctx) diff --git a/daemon/config.go b/daemon/config.go index cd0ebcef..86263d6a 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -4,7 +4,6 @@ import ( "context" "github.com/AliyunContainerService/terway/pkg/aliyun/client" - "github.com/AliyunContainerService/terway/pkg/aliyun/instance" "github.com/AliyunContainerService/terway/pkg/k8s" "github.com/AliyunContainerService/terway/pkg/utils" "github.com/AliyunContainerService/terway/pkg/vswitch" @@ -25,7 +24,7 @@ func getDynamicConfig(ctx context.Context, k8s k8s.Kubernetes) (string, string, return cfg, label, err } -func getENIConfig(cfg *daemon.Config) *types.ENIConfig { +func getENIConfig(cfg *daemon.Config) *daemon.ENIConfig { vswitchSelectionPolicy := vswitch.VSwitchSelectionPolicyRandom switch cfg.VSwitchSelectionPolicy { case "ordered": @@ -33,18 +32,16 @@ func getENIConfig(cfg *daemon.Config) *types.ENIConfig { vswitchSelectionPolicy = vswitch.VSwitchSelectionPolicyMost } - eniSelectionPolicy := types.EniSelectionPolicyMostIPs + eniSelectionPolicy := daemon.EniSelectionPolicyMostIPs switch cfg.EniSelectionPolicy { case "least_ips": - eniSelectionPolicy = types.EniSelectionPolicyLeastIPs + eniSelectionPolicy = daemon.EniSelectionPolicyLeastIPs } - eniConfig := &types.ENIConfig{ - ZoneID: instance.GetInstanceMeta().ZoneID, + eniConfig := &daemon.ENIConfig{ VSwitchOptions: nil, ENITags: cfg.ENITags, SecurityGroupIDs: cfg.GetSecurityGroups(), - InstanceID: instance.GetInstanceMeta().InstanceID, VSwitchSelectionPolicy: vswitchSelectionPolicy, EniSelectionPolicy: eniSelectionPolicy, ResourceGroupID: cfg.ResourceGroupID, @@ -59,24 +56,20 @@ func getENIConfig(cfg *daemon.Config) *types.ENIConfig { } } - if len(eniConfig.VSwitchOptions) == 0 { - eniConfig.VSwitchOptions = []string{instance.GetInstanceMeta().VSwitchID} - } - if cfg.EnableENITrunking { - types.EnableFeature(&eniConfig.EniTypeAttr, types.FeatTrunk) + daemon.EnableFeature(&eniConfig.EniTypeAttr, daemon.FeatTrunk) } if cfg.EnableERDMA { - types.EnableFeature(&eniConfig.EniTypeAttr, types.FeatERDMA) + daemon.EnableFeature(&eniConfig.EniTypeAttr, daemon.FeatERDMA) } return eniConfig } // the actual size for pool is minIdle and maxIdle -func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *client.Limits) (*types.PoolConfig, error) { +func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *client.Limits) (*daemon.PoolConfig, error) { - poolConfig := &types.PoolConfig{ + poolConfig := &daemon.PoolConfig{ BatchSize: 10, } diff --git a/daemon/config_test.go b/daemon/config_test.go index 3e85d230..3f914892 100644 --- a/daemon/config_test.go +++ b/daemon/config_test.go @@ -8,25 +8,51 @@ import ( "github.com/AliyunContainerService/terway/pkg/aliyun/client" "github.com/AliyunContainerService/terway/pkg/aliyun/instance" "github.com/AliyunContainerService/terway/pkg/vswitch" - "github.com/AliyunContainerService/terway/types" "github.com/AliyunContainerService/terway/types/daemon" ) -func init() { - instance.SetPopulateFunc(func() *instance.Instance { - return &instance.Instance{ - RegionID: "regionID", - ZoneID: "zoneID", - VPCID: "vpc", - VSwitchID: "vsw", - PrimaryMAC: "", - InstanceID: "instanceID", - InstanceType: "", - } - }) +type Mock struct { + regionID string + zoneID string + vSwitchID string + primaryMAC string + instanceID string + instanceType string +} + +func (m *Mock) GetRegionID() (string, error) { + return m.regionID, nil +} + +func (m *Mock) GetZoneID() (string, error) { + return m.zoneID, nil +} + +func (m *Mock) GetVSwitchID() (string, error) { + return m.vSwitchID, nil +} + +func (m *Mock) GetPrimaryMAC() (string, error) { + return m.primaryMAC, nil +} + +func (m *Mock) GetInstanceID() (string, error) { + return m.instanceID, nil +} + +func (m *Mock) GetInstanceType() (string, error) { + return m.instanceType, nil } func TestGetPoolConfigWithENIMultiIPMode(t *testing.T) { + instance.Init(&Mock{ + regionID: "regionID", + zoneID: "zoneID", + vSwitchID: "vsw", + primaryMAC: "", + instanceID: "instanceID", + instanceType: "", + }) cfg := &daemon.Config{ MaxPoolSize: 5, MinPoolSize: 1, @@ -46,6 +72,14 @@ func TestGetPoolConfigWithENIMultiIPMode(t *testing.T) { } func TestGetENIConfig(t *testing.T) { + instance.Init(&Mock{ + regionID: "regionID", + zoneID: "zoneID", + vSwitchID: "vsw", + primaryMAC: "", + instanceID: "instanceID", + instanceType: "", + }) cfg := &daemon.Config{ ENITags: map[string]string{"aa": "bb"}, SecurityGroups: []string{"sg1", "sg2"}, @@ -61,13 +95,10 @@ func TestGetENIConfig(t *testing.T) { eniConfig := getENIConfig(cfg) - assert.Equal(t, "zoneID", eniConfig.ZoneID) - assert.Equal(t, []string{"vswitch1", "vswitch2"}, eniConfig.VSwitchOptions) assert.Equal(t, 1, len(eniConfig.ENITags)) assert.Equal(t, []string{"sg1", "sg2"}, eniConfig.SecurityGroupIDs) - assert.Equal(t, "instanceID", eniConfig.InstanceID) assert.Equal(t, vswitch.VSwitchSelectionPolicyMost, eniConfig.VSwitchSelectionPolicy) - assert.Equal(t, types.EniSelectionPolicyMostIPs, eniConfig.EniSelectionPolicy) + assert.Equal(t, daemon.EniSelectionPolicyMostIPs, eniConfig.EniSelectionPolicy) assert.Equal(t, "rgID", eniConfig.ResourceGroupID) - assert.Equal(t, types.Feat(3), eniConfig.EniTypeAttr) + assert.Equal(t, daemon.Feat(3), eniConfig.EniTypeAttr) } diff --git a/daemon/daemon.go b/daemon/daemon.go index 70e07ab4..0e9c85af 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -850,7 +850,7 @@ func checkInstance(limit *client.Limits, daemonMode string, config *daemon.Confi } // initTrunk to ensure trunk eni is present. Return eni id if found. -func initTrunk(config *daemon.Config, poolConfig *types.PoolConfig, k8sClient k8s.Kubernetes, f factory.Factory) (string, error) { +func initTrunk(config *daemon.Config, poolConfig *daemon.PoolConfig, k8sClient k8s.Kubernetes, f factory.Factory) (string, error) { var err error // get eni id form node annotation @@ -909,7 +909,7 @@ func initTrunk(config *daemon.Config, poolConfig *types.PoolConfig, k8sClient k8 return trunk.ID, nil } -func runDevicePlugin(daemonMode string, config *daemon.Config, poolConfig *types.PoolConfig) { +func runDevicePlugin(daemonMode string, config *daemon.Config, poolConfig *daemon.PoolConfig) { switch daemonMode { case daemon.ModeENIMultiIP: if config.EnableENITrunking { diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index f2fa36dd..18d3306f 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -86,7 +86,7 @@ func Test_checkInstance1(t *testing.T) { func Test_initTrunk(t *testing.T) { type args struct { config *daemon.Config - poolConfig *types.PoolConfig + poolConfig *daemon.PoolConfig k8sClient *k8smocks.Kubernetes f *factorymocks.Factory } @@ -105,7 +105,7 @@ func Test_initTrunk(t *testing.T) { EnableENITrunking: true, EnableERDMA: true, }, - poolConfig: &types.PoolConfig{ + poolConfig: &daemon.PoolConfig{ MaxENI: 2, }, k8sClient: k8smocks.NewKubernetes(t), @@ -148,7 +148,7 @@ func Test_initTrunk(t *testing.T) { EnableENITrunking: true, EnableERDMA: true, }, - poolConfig: &types.PoolConfig{ + poolConfig: &daemon.PoolConfig{ MaxENI: 2, }, k8sClient: k8smocks.NewKubernetes(t), @@ -180,7 +180,7 @@ func Test_initTrunk(t *testing.T) { EnableENITrunking: true, EnableERDMA: true, }, - poolConfig: &types.PoolConfig{ + poolConfig: &daemon.PoolConfig{ MaxENI: 2, }, k8sClient: k8smocks.NewKubernetes(t), diff --git a/deploy/images/terway-controlplane/Dockerfile b/deploy/images/terway-controlplane/Dockerfile index 5d86444a..1cc29563 100644 --- a/deploy/images/terway-controlplane/Dockerfile +++ b/deploy/images/terway-controlplane/Dockerfile @@ -3,7 +3,7 @@ ARG TERWAY_POLICY_IMAGE=registry-cn-zhangjiakou.ack.aliyuncs.com/acs/terway:poli FROM --platform=$TARGETPLATFORM ${TERWAY_POLICY_IMAGE} AS policy-dist -FROM --platform=$BUILDPLATFORM golang:1.23.2 AS builder +FROM --platform=$BUILDPLATFORM golang:1.24.0 AS builder ARG GOPROXY ARG TARGETOS ARG TARGETARCH diff --git a/deploy/images/terway/Dockerfile b/deploy/images/terway/Dockerfile index 2c9363a9..1f512d89 100644 --- a/deploy/images/terway/Dockerfile +++ b/deploy/images/terway/Dockerfile @@ -12,7 +12,7 @@ FROM --platform=$TARGETPLATFORM ${CILIUM_BPFTOOL_IMAGE} AS bpftool-dist FROM --platform=$TARGETPLATFORM ${CILIUM_IPROUTE2_IMAGE} AS iproute2-dist FROM --platform=$TARGETPLATFORM ${CILIUM_IPTABLES_IMAGE} AS iptables-dist -FROM --platform=$BUILDPLATFORM golang:1.23.2 AS builder +FROM --platform=$BUILDPLATFORM golang:1.24.0 AS builder ARG GOPROXY ARG TARGETOS ARG TARGETARCH diff --git a/examples/maxpods/maxpods.go b/examples/maxpods/maxpods.go index d6ad47f3..82c6d1b9 100644 --- a/examples/maxpods/maxpods.go +++ b/examples/maxpods/maxpods.go @@ -32,15 +32,21 @@ func init() { func main() { flag.Parse() log.SetOutput(io.Discard) - ins := instance.GetInstanceMeta() - + regionID, err := instance.GetInstanceMeta().GetRegionID() + if err != nil { + panic(err) + } + instanceType, err := instance.GetInstanceMeta().GetInstanceType() + if err != nil { + panic(err) + } providers := []credential.Interface{ credential.NewAKPairProvider(accessKeyID, accessKeySecret), credential.NewEncryptedCredentialProvider(credentialPath), credential.NewMetadataProvider(), } - c, err := credential.NewClientMgr(ins.RegionID, providers...) + c, err := credential.NewClientMgr(regionID, providers...) if err != nil { panic(err) } @@ -51,13 +57,13 @@ func main() { } if mode == "terway-eniip" { - limit, err := client.LimitProviders["ecs"].GetLimit(api, instance.GetInstanceMeta().InstanceType) + limit, err := client.LimitProviders["ecs"].GetLimit(api, instanceType) if err != nil { panic(err) } fmt.Println(limit.IPv4PerAdapter * (limit.Adapters - 1)) } else if mode == "terway-eni" { - limit, err := client.LimitProviders["ecs"].GetLimit(api, instance.GetInstanceMeta().InstanceType) + limit, err := client.LimitProviders["ecs"].GetLimit(api, instanceType) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index 2cdffbf2..32c03166 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/AliyunContainerService/terway -go 1.22.3 +go 1.23.0 -toolchain go1.23.3 +toolchain go1.24.0 require ( github.com/Jeffail/gabs/v2 v2.7.0 @@ -14,50 +14,49 @@ require ( github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.3.0 github.com/denverdino/aliyungo v0.0.0-20201215054313-f635de23c5e0 - github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7 github.com/evanphx/json-patch v5.6.0+incompatible github.com/go-logr/logr v1.4.2 github.com/go-playground/mold/v4 v4.2.0 github.com/go-playground/validator/v10 v10.11.1 github.com/google/uuid v1.6.0 - github.com/onsi/ginkgo/v2 v2.19.0 - github.com/onsi/gomega v1.33.1 + github.com/onsi/ginkgo/v2 v2.21.0 + github.com/onsi/gomega v1.35.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 github.com/pterm/pterm v0.12.62 github.com/samber/lo v1.39.0 github.com/spf13/cobra v1.8.1 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/vishvananda/netlink v1.2.1-beta.2 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/trace v1.28.0 - go.uber.org/atomic v1.10.0 + go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs v1.6.0 - golang.org/x/mod v0.17.0 - golang.org/x/net v0.26.0 - golang.org/x/sync v0.7.0 - golang.org/x/sys v0.21.0 - golang.org/x/time v0.3.0 + golang.org/x/mod v0.21.0 + golang.org/x/net v0.35.0 + golang.org/x/sync v0.11.0 + golang.org/x/sys v0.30.0 + golang.org/x/time v0.7.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/grpc v1.65.0 - google.golang.org/protobuf v1.34.2 + google.golang.org/protobuf v1.35.1 gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.31.1 - k8s.io/apiextensions-apiserver v0.31.0 - k8s.io/apimachinery v0.31.1 - k8s.io/apiserver v0.31.0 - k8s.io/client-go v0.31.1 - k8s.io/code-generator v0.31.0 - k8s.io/component-base v0.31.1 + k8s.io/api v0.32.2 + k8s.io/apiextensions-apiserver v0.32.2 + k8s.io/apimachinery v0.32.2 + k8s.io/apiserver v0.32.2 + k8s.io/client-go v0.32.2 + k8s.io/code-generator v0.32.2 + k8s.io/component-base v0.32.2 k8s.io/klog/v2 v2.130.1 k8s.io/kubelet v0.29.9 - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 - sigs.k8s.io/controller-runtime v0.19.0 - sigs.k8s.io/e2e-framework v0.5.0 + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 + sigs.k8s.io/controller-runtime v0.20.0 + sigs.k8s.io/e2e-framework v0.6.0 sigs.k8s.io/yaml v1.4.0 ) @@ -79,23 +78,23 @@ require ( github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/gookit/color v1.5.3 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect - github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -104,8 +103,7 @@ require ( github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/mattn/go-shellwords v1.0.12 // indirect - github.com/moby/spdystream v0.4.0 // indirect + github.com/moby/spdystream v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -123,29 +121,29 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/vishvananda/netns v0.0.4 // indirect - github.com/vladimirvivien/gexe v0.3.0 // indirect + github.com/vladimirvivien/gexe v0.4.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect - golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.33.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/tools v0.26.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect ) replace github.com/vishvananda/netlink => github.com/BSWANG/netlink v1.0.1-0.20220803105814-1f63f9d61229 diff --git a/go.sum b/go.sum index d3ef4ce7..4628b828 100644 --- a/go.sum +++ b/go.sum @@ -282,7 +282,6 @@ github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7 h1:Cvj7S8I4Xpx78KAl6TwTmMHuHlZ/0SM60NUneGJQ7IE= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= @@ -346,8 +345,9 @@ github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= @@ -359,8 +359,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -426,6 +426,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -455,8 +457,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -504,8 +506,6 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= -github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -577,8 +577,6 @@ github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWV github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= @@ -588,8 +586,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= -github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= @@ -626,8 +624,8 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -637,8 +635,8 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -801,8 +799,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -822,8 +820,8 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= -github.com/vladimirvivien/gexe v0.3.0 h1:4xwiOwGrDob5OMR6E92B9olDXYDglXdHhzR1ggYtWJM= -github.com/vladimirvivien/gexe v0.3.0/go.mod h1:fp7cy60ON1xjhtEI/+bfSEIXX35qgmI+iRYlGOqbBFM= +github.com/vladimirvivien/gexe v0.4.1 h1:W9gWkp8vSPjDoXDu04Yp4KljpVMaSt8IQuHswLDd5LY= +github.com/vladimirvivien/gexe v0.4.1/go.mod h1:3gjgTqE2c0VyHnU5UOIwk7gyNzZDGulPb/DJPgcw64E= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -873,8 +871,8 @@ go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -883,8 +881,8 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -900,8 +898,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -915,8 +913,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -941,8 +939,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -987,15 +985,15 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1009,8 +1007,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1088,15 +1086,15 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1107,16 +1105,16 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1166,8 +1164,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1222,10 +1220,10 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1259,8 +1257,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1313,33 +1311,33 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= -k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= -k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= -k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= +k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= +k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= +k8s.io/apiextensions-apiserver v0.32.2 h1:2YMk285jWMk2188V2AERy5yDwBYrjgWYggscghPCvV4= +k8s.io/apiextensions-apiserver v0.32.2/go.mod h1:GPwf8sph7YlJT3H6aKUWtd0E+oyShk/YHWQHf/OOgCA= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= -k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= +k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.31.0 h1:p+2dgJjy+bk+B1Csz+mc2wl5gHwvNkC9QJV+w55LVrY= -k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk= +k8s.io/apiserver v0.32.2 h1:WzyxAu4mvLkQxwD9hGa4ZfExo3yZZaYzoYvvVDlM6vw= +k8s.io/apiserver v0.32.2/go.mod h1:PEwREHiHNU2oFdte7BjzA1ZyjWjuckORLIK/wLV5goM= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= -k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= +k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= +k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= -k8s.io/code-generator v0.31.0 h1:w607nrMi1KeDKB3/F/J4lIoOgAwc+gV9ZKew4XRfMp8= -k8s.io/code-generator v0.31.0/go.mod h1:84y4w3es8rOJOUUP1rLsIiGlO1JuEaPFXQPA9e/K6U0= +k8s.io/code-generator v0.32.2 h1:CIvyPrLWP7cMgrqval2qYT839YAwCDeSvGfXgWSNpHQ= +k8s.io/code-generator v0.32.2/go.mod h1:plh7bWk7JztAUkHM4zpbdy0KOMdrhsePcZL2HLWFH7Y= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8= -k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w= +k8s.io/component-base v0.32.2 h1:1aUL5Vdmu7qNo4ZsE+569PV5zFatM9hl+lb3dEea2zU= +k8s.io/component-base v0.32.2/go.mod h1:PXJ61Vx9Lg+P5mS8TLd7bCIr+eMJRQTyXe8KvkrvJq0= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= @@ -1347,8 +1345,8 @@ k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 h1:NGrVE502P0s0/1hudf8zjgwki1X/TByhmAoILTarmzo= -k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= +k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 h1:si3PfKm8dDYxgfbeA6orqrtLkvvIeH8UqffFJDl0bz4= +k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= @@ -1356,31 +1354,31 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= k8s.io/kubelet v0.29.9 h1:Qbnz4otarQi5E8Z80Y3Y8AY5wfyc6WQjUQ6hU302gPQ= k8s.io/kubelet v0.29.9/go.mod h1:jOTCkSUkzTu6t5SvxcSDAg3n4bZy3+mCOe87WJ3NS58= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= -sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= -sigs.k8s.io/e2e-framework v0.5.0 h1:YLhk8R7EHuTFQAe6Fxy5eBzn5Vb+yamR5u8MH1Rq3cE= -sigs.k8s.io/e2e-framework v0.5.0/go.mod h1:jJSH8u2RNmruekUZgHAtmRjb5Wj67GErli9UjLSY7Zc= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/controller-runtime v0.20.0 h1:jjkMo29xEXH+02Md9qaVXfEIaMESSpy3TBWPrsfQkQs= +sigs.k8s.io/controller-runtime v0.20.0/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU= +sigs.k8s.io/e2e-framework v0.6.0 h1:p7hFzHnLKO7eNsWGI2AbC1Mo2IYxidg49BiT4njxkrM= +sigs.k8s.io/e2e-framework v0.6.0/go.mod h1:IREnCHnKgRCioLRmNi0hxSJ1kJ+aAdjEKK/gokcZu4k= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/pkg/aliyun/client/evc_2.go b/pkg/aliyun/client/ecs_2.go similarity index 100% rename from pkg/aliyun/client/evc_2.go rename to pkg/aliyun/client/ecs_2.go diff --git a/pkg/aliyun/client/mocks/ECS.go b/pkg/aliyun/client/mocks/ECS.go index b5cf92a6..3ddcad1f 100644 --- a/pkg/aliyun/client/mocks/ECS.go +++ b/pkg/aliyun/client/mocks/ECS.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.52.2. DO NOT EDIT. package mocks diff --git a/pkg/aliyun/client/mocks/EFLO.go b/pkg/aliyun/client/mocks/EFLO.go index 942bd31e..f4bc3364 100644 --- a/pkg/aliyun/client/mocks/EFLO.go +++ b/pkg/aliyun/client/mocks/EFLO.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.52.2. DO NOT EDIT. package mocks diff --git a/pkg/aliyun/client/mocks/LimitProvider.go b/pkg/aliyun/client/mocks/LimitProvider.go index 8e1de76d..e7ebb0e3 100644 --- a/pkg/aliyun/client/mocks/LimitProvider.go +++ b/pkg/aliyun/client/mocks/LimitProvider.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.52.2. DO NOT EDIT. package mocks @@ -42,6 +42,36 @@ func (_m *LimitProvider) GetLimit(_a0 interface{}, instanceType string) (*client return r0, r1 } +// GetLimitFromAnno provides a mock function with given fields: anno +func (_m *LimitProvider) GetLimitFromAnno(anno map[string]string) (*client.Limits, error) { + ret := _m.Called(anno) + + if len(ret) == 0 { + panic("no return value specified for GetLimitFromAnno") + } + + var r0 *client.Limits + var r1 error + if rf, ok := ret.Get(0).(func(map[string]string) (*client.Limits, error)); ok { + return rf(anno) + } + if rf, ok := ret.Get(0).(func(map[string]string) *client.Limits); ok { + r0 = rf(anno) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*client.Limits) + } + } + + if rf, ok := ret.Get(1).(func(map[string]string) error); ok { + r1 = rf(anno) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // NewLimitProvider creates a new instance of LimitProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewLimitProvider(t interface { diff --git a/pkg/aliyun/client/mocks/VPC.go b/pkg/aliyun/client/mocks/VPC.go index 558d7ca8..6b1c51e8 100644 --- a/pkg/aliyun/client/mocks/VPC.go +++ b/pkg/aliyun/client/mocks/VPC.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.52.2. DO NOT EDIT. package mocks diff --git a/pkg/aliyun/credential/aliyun_client_mgr.go b/pkg/aliyun/credential/aliyun_client_mgr.go index 5be96410..81bb8fcd 100644 --- a/pkg/aliyun/credential/aliyun_client_mgr.go +++ b/pkg/aliyun/credential/aliyun_client_mgr.go @@ -86,6 +86,8 @@ type ClientMgr struct { vpcDomainOverride string efloDomainOverride string + efloRegionOverride string + endpointType string } @@ -109,6 +111,11 @@ func NewClientMgr(regionID string, providers ...Interface) (*ClientMgr, error) { return nil, err } + mgr.efloRegionOverride = os.Getenv("EFLO_REGION_ID") + if mgr.efloRegionOverride == "" { + mgr.efloRegionOverride = regionID + } + mgr.endpointType = "vpc" if os.Getenv("ALICLOUD_ENDPOINT_TYPE") != "" { mgr.endpointType = os.Getenv("ALICLOUD_ENDPOINT_TYPE") @@ -205,7 +212,7 @@ func (c *ClientMgr) refreshToken() (bool, error) { c.vpc.Domain = c.vpcDomainOverride } - c.eflo, err = eflo.NewClientWithOptions(c.regionID, clientCfg(), cc.Credential) + c.eflo, err = eflo.NewClientWithOptions(c.efloRegionOverride, clientCfg(), cc.Credential) if err != nil { return false, err } diff --git a/pkg/aliyun/eni/eni.go b/pkg/aliyun/eni/eni.go index 29d166ac..4d0a8f62 100644 --- a/pkg/aliyun/eni/eni.go +++ b/pkg/aliyun/eni/eni.go @@ -103,7 +103,10 @@ func (e *ENIMetadata) GetENIPrivateIPv6AddressesByMACv2(mac string) ([]netip.Add func (e *ENIMetadata) GetENIs(containsMainENI bool) ([]*daemon.ENI, error) { var enis []*daemon.ENI - mainENIMac := instance.GetInstanceMeta().PrimaryMAC + mainENIMac, err := instance.GetInstanceMeta().GetPrimaryMAC() + if err != nil { + return nil, err + } macs, err := metadata.GetENIsMAC() if err != nil { diff --git a/pkg/aliyun/instance/ecs.go b/pkg/aliyun/instance/ecs.go index 9dfd0e41..decec7e7 100644 --- a/pkg/aliyun/instance/ecs.go +++ b/pkg/aliyun/instance/ecs.go @@ -1,48 +1,88 @@ package instance import ( - "fmt" + "sync/atomic" "github.com/AliyunContainerService/terway/pkg/aliyun/metadata" ) -func DefaultPopulate() *Instance { +type ECS struct { + regionID atomic.Value + zoneID atomic.Value + vSwitchID atomic.Value + primaryMAC atomic.Value + instanceID atomic.Value + instanceType atomic.Value +} + +func (e *ECS) GetRegionID() (string, error) { + if v := e.regionID.Load(); v != nil { + return v.(string), nil + } regionID, err := metadata.GetLocalRegion() - if err != nil || regionID == "" { - panic(fmt.Errorf("error get regionID %w", err)) + if err != nil { + return "", err + } + e.regionID.Store(regionID) + return regionID, nil +} + +func (e *ECS) GetZoneID() (string, error) { + if v := e.zoneID.Load(); v != nil { + return v.(string), nil } zoneID, err := metadata.GetLocalZone() - if err != nil || zoneID == "" { - panic(fmt.Errorf("error get zoneID %w", err)) + if err != nil { + return "", err } - vpcID, err := metadata.GetLocalVPC() - if err != nil || vpcID == "" { - panic(fmt.Errorf("error get vpcID %w", err)) + e.zoneID.Store(zoneID) + return zoneID, nil +} + +func (e *ECS) GetVSwitchID() (string, error) { + if v := e.vSwitchID.Load(); v != nil { + return v.(string), nil } - instanceID, err := metadata.GetLocalInstanceID() - if err != nil || instanceID == "" { - panic(fmt.Errorf("error get instanceID %w", err)) + vSwitchID, err := metadata.GetLocalVswitch() + if err != nil { + return "", err } - instanceType, err := metadata.GetInstanceType() - if err != nil || instanceType == "" { - panic(fmt.Errorf("error get instanceType %w", err)) + e.vSwitchID.Store(vSwitchID) + return vSwitchID, nil +} + +func (e *ECS) GetPrimaryMAC() (string, error) { + if v := e.primaryMAC.Load(); v != nil { + return v.(string), nil } - vSwitchID, err := metadata.GetLocalVswitch() - if err != nil || vSwitchID == "" { - panic(fmt.Errorf("error get vSwitchID %w", err)) + primaryMAC, err := metadata.GetPrimaryENIMAC() + if err != nil { + return "", err } - mac, err := metadata.GetPrimaryENIMAC() + e.primaryMAC.Store(primaryMAC) + return primaryMAC, nil +} + +func (e *ECS) GetInstanceID() (string, error) { + if v := e.instanceID.Load(); v != nil { + return v.(string), nil + } + instanceID, err := metadata.GetLocalInstanceID() if err != nil { - panic(fmt.Errorf("error get eth0's mac %w", err)) + return "", err } + e.instanceID.Store(instanceID) + return instanceID, nil +} - return &Instance{ - RegionID: regionID, - ZoneID: zoneID, - VPCID: vpcID, - VSwitchID: vSwitchID, - InstanceID: instanceID, - InstanceType: instanceType, - PrimaryMAC: mac, +func (e *ECS) GetInstanceType() (string, error) { + if v := e.instanceType.Load(); v != nil { + return v.(string), nil + } + instanceType, err := metadata.GetInstanceType() + if err != nil { + return "", err } + e.instanceType.Store(instanceType) + return instanceType, nil } diff --git a/pkg/aliyun/instance/eflo.go b/pkg/aliyun/instance/eflo.go new file mode 100644 index 00000000..aad79d2c --- /dev/null +++ b/pkg/aliyun/instance/eflo.go @@ -0,0 +1,104 @@ +package instance + +import ( + "encoding/json" + "errors" + "os" + "sync" +) + +var ErrUnSupport = errors.New("unsupported") + +type EFLO struct { + f *efloConfig + mutex sync.Mutex +} + +func (e *EFLO) GetRegionID() (string, error) { + e.mutex.Lock() + defer e.mutex.Unlock() + if e.f == nil { + cfg, err := loadEfloConfig() + if err != nil { + return "", err + } + e.f = cfg + } + return e.f.RegionID, nil +} + +func (e *EFLO) GetZoneID() (string, error) { + e.mutex.Lock() + defer e.mutex.Unlock() + if e.f == nil { + cfg, err := loadEfloConfig() + if err != nil { + return "", err + } + e.f = cfg + } + return e.f.ZoneID, nil +} + +func (e *EFLO) GetVSwitchID() (string, error) { + e.mutex.Lock() + defer e.mutex.Unlock() + if e.f == nil { + cfg, err := loadEfloConfig() + if err != nil { + return "", err + } + e.f = cfg + } + return e.f.AckNicName, nil +} + +func (e *EFLO) GetPrimaryMAC() (string, error) { + return "", ErrUnSupport +} + +func (e *EFLO) GetInstanceID() (string, error) { + e.mutex.Lock() + defer e.mutex.Unlock() + if e.f == nil { + cfg, err := loadEfloConfig() + if err != nil { + return "", err + } + e.f = cfg + } + return e.f.NodeID, nil +} + +func (e *EFLO) GetInstanceType() (string, error) { + e.mutex.Lock() + defer e.mutex.Unlock() + if e.f == nil { + cfg, err := loadEfloConfig() + if err != nil { + return "", err + } + e.f = cfg + } + return e.f.InstanceType, nil +} + +func loadEfloConfig() (*efloConfig, error) { + file, err := os.ReadFile("/etc/eflo_config/lingjun_config") + if err != nil { + return nil, err + } + + cfg := &efloConfig{} + err = json.Unmarshal(file, cfg) + + return cfg, err +} + +type efloConfig struct { + RegionID string `json:"RegionId"` + ZoneID string `json:"ZoneId"` + NodeID string `json:"NodeId"` + InstanceType string `json:"InstanceType"` + AckNicName string `json:"AckNicName"` +} diff --git a/pkg/aliyun/instance/eflo_linux.go b/pkg/aliyun/instance/eflo_linux.go deleted file mode 100644 index 00ce8436..00000000 --- a/pkg/aliyun/instance/eflo_linux.go +++ /dev/null @@ -1,40 +0,0 @@ -package instance - -import ( - "encoding/json" - "os" -) - -func EfloPopulate() *Instance { - cfg := loadEfloConfig() - - return &Instance{ - RegionID: cfg.RegionID, - InstanceType: cfg.InstanceType, - InstanceID: cfg.NodeID, - ZoneID: cfg.ZoneID, - } -} - -func loadEfloConfig() *efloConfig { - file, err := os.ReadFile("/etc/eflo_config/lingjun_config") - if err != nil { - panic(err) - } - - cfg := &efloConfig{} - err = json.Unmarshal(file, cfg) - if err != nil { - panic(err) - } - - return cfg -} - -type efloConfig struct { - RegionID string `json:"RegionId"` - ZoneID string `json:"ZoneId"` - NodeID string `json:"NodeId"` - InstanceType string `json:"InstanceType"` - AckNicName string `json:"AckNicName"` -} diff --git a/pkg/aliyun/instance/eflo_test.go b/pkg/aliyun/instance/eflo_test.go new file mode 100644 index 00000000..0c6b5d41 --- /dev/null +++ b/pkg/aliyun/instance/eflo_test.go @@ -0,0 +1,34 @@ +package instance + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEFLO(t *testing.T) { + e := EFLO{ + f: &efloConfig{ + RegionID: "a", + ZoneID: "b", + NodeID: "c", + InstanceType: "d", + AckNicName: "e", + }, + } + regionID, err := e.GetRegionID() + assert.Nil(t, err) + assert.Equal(t, "a", regionID) + + zoneID, err := e.GetZoneID() + assert.Nil(t, err) + assert.Equal(t, "b", zoneID) + + nodeID, err := e.GetInstanceID() + assert.Nil(t, err) + assert.Equal(t, "c", nodeID) + + instanceType, err := e.GetInstanceType() + assert.Nil(t, err) + assert.Equal(t, "d", instanceType) +} diff --git a/pkg/aliyun/instance/eflo_unlinux.go b/pkg/aliyun/instance/eflo_unlinux.go deleted file mode 100644 index bf953487..00000000 --- a/pkg/aliyun/instance/eflo_unlinux.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !linux - -package instance - -func EfloPopulate() *Instance { - return &Instance{} -} diff --git a/pkg/aliyun/instance/instance.go b/pkg/aliyun/instance/instance.go index 8e19ac72..7d557b4d 100644 --- a/pkg/aliyun/instance/instance.go +++ b/pkg/aliyun/instance/instance.go @@ -1,43 +1,22 @@ package instance -import ( - "sync" - - "k8s.io/klog/v2" -) - -var defaultIns *Instance -var once sync.Once - -type PopulateFunc func() *Instance - -var populate PopulateFunc - -type Instance struct { - RegionID string - ZoneID string - VPCID string - VSwitchID string - PrimaryMAC string - - InstanceID string - InstanceType string +//go:generate mockery --name Interface + +type Interface interface { + GetRegionID() (string, error) + GetZoneID() (string, error) + GetVSwitchID() (string, error) + GetPrimaryMAC() (string, error) + GetInstanceID() (string, error) + GetInstanceType() (string, error) } -func init() { - populate = DefaultPopulate -} +var defaultIns Interface = &ECS{} -func SetPopulateFunc(fn PopulateFunc) { - populate = fn +func Init(in Interface) { + defaultIns = in } -func GetInstanceMeta() *Instance { - once.Do(func() { - defaultIns = populate() - - klog.Infof("load instance metadata %#v", defaultIns) - }) - +func GetInstanceMeta() Interface { return defaultIns } diff --git a/pkg/aliyun/instance/mocks/Interface.go b/pkg/aliyun/instance/mocks/Interface.go new file mode 100644 index 00000000..3b6e8f6c --- /dev/null +++ b/pkg/aliyun/instance/mocks/Interface.go @@ -0,0 +1,192 @@ +// Code generated by mockery v2.52.2. DO NOT EDIT. + +package mocks + +import mock "github.com/stretchr/testify/mock" + +// Interface is an autogenerated mock type for the Interface type +type Interface struct { + mock.Mock +} + +// GetInstanceID provides a mock function with no fields +func (_m *Interface) GetInstanceID() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetInstanceID") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetInstanceType provides a mock function with no fields +func (_m *Interface) GetInstanceType() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetInstanceType") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetPrimaryMAC provides a mock function with no fields +func (_m *Interface) GetPrimaryMAC() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetPrimaryMAC") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetRegionID provides a mock function with no fields +func (_m *Interface) GetRegionID() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetRegionID") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetVSwitchID provides a mock function with no fields +func (_m *Interface) GetVSwitchID() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetVSwitchID") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetZoneID provides a mock function with no fields +func (_m *Interface) GetZoneID() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetZoneID") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewInterface creates a new instance of Interface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewInterface(t interface { + mock.TestingT + Cleanup(func()) +}) *Interface { + mock := &Interface{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/controller/common/types_default.go b/pkg/controller/common/types_default.go index a240de1d..ccaf473b 100644 --- a/pkg/controller/common/types_default.go +++ b/pkg/controller/common/types_default.go @@ -24,11 +24,17 @@ import ( corev1 "k8s.io/api/core/v1" + "github.com/AliyunContainerService/terway/pkg/utils" "github.com/AliyunContainerService/terway/types" ) func NewNodeInfo(node *corev1.Node) (*NodeInfo, error) { res := &NodeInfo{NodeName: node.Name} + + if utils.ISLinJunNode(node.Labels) { + return res, nil + } + ids := strings.Split(node.Spec.ProviderID, ".") if len(ids) < 2 { return nil, fmt.Errorf("error parse providerID %s", node.Spec.ProviderID) diff --git a/pkg/controller/common/types_default_test.go b/pkg/controller/common/types_default_test.go new file mode 100644 index 00000000..37ce8fe1 --- /dev/null +++ b/pkg/controller/common/types_default_test.go @@ -0,0 +1,111 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/AliyunContainerService/terway/types" +) + +func TestNodeInfoIsCreatedSuccessfully(t *testing.T) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + corev1.LabelTopologyRegion: "test-region", + corev1.LabelInstanceTypeStable: "test-instance-type", + corev1.LabelTopologyZone: "test-zone", + }, + Annotations: map[string]string{ + types.TrunkOn: "test-trunk-eni-id", + }, + }, + Spec: corev1.NodeSpec{ + ProviderID: "provider.test-instance-id", + }, + } + + nodeInfo, err := NewNodeInfo(node) + assert.Nil(t, err) + assert.Equal(t, "test-node", nodeInfo.NodeName) + assert.Equal(t, "test-instance-id", nodeInfo.InstanceID) + assert.Equal(t, "test-trunk-eni-id", nodeInfo.TrunkENIID) + assert.Equal(t, "test-region", nodeInfo.RegionID) + assert.Equal(t, "test-instance-type", nodeInfo.InstanceType) + assert.Equal(t, "test-zone", nodeInfo.ZoneID) +} + +func TestNodeInfoReturnsErrorWhenProviderIDIsInvalid(t *testing.T) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Spec: corev1.NodeSpec{ + ProviderID: "invalid-provider-id", + }, + } + + nodeInfo, err := NewNodeInfo(node) + assert.NotNil(t, err) + assert.Nil(t, nodeInfo) +} + +func TestNodeInfoReturnsErrorWhenRegionIDIsMissing(t *testing.T) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + corev1.LabelInstanceTypeStable: "test-instance-type", + corev1.LabelTopologyZone: "test-zone", + }, + }, + Spec: corev1.NodeSpec{ + ProviderID: "provider.test-instance-id", + }, + } + + nodeInfo, err := NewNodeInfo(node) + assert.NotNil(t, err) + assert.Nil(t, nodeInfo) +} + +func TestNodeInfoReturnsErrorWhenInstanceTypeIsMissing(t *testing.T) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + corev1.LabelTopologyRegion: "test-region", + corev1.LabelTopologyZone: "test-zone", + }, + }, + Spec: corev1.NodeSpec{ + ProviderID: "provider.test-instance-id", + }, + } + + nodeInfo, err := NewNodeInfo(node) + assert.NotNil(t, err) + assert.Nil(t, nodeInfo) +} + +func TestNodeInfoReturnsErrorWhenZoneLabelIsMissing(t *testing.T) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + corev1.LabelTopologyRegion: "test-region", + corev1.LabelInstanceTypeStable: "test-instance-type", + }, + }, + Spec: corev1.NodeSpec{ + ProviderID: "provider.test-instance-id", + }, + } + + nodeInfo, err := NewNodeInfo(node) + assert.NotNil(t, err) + assert.Nil(t, nodeInfo) +} diff --git a/pkg/controller/mocks/Interface.go b/pkg/controller/mocks/Interface.go index ff1d4366..175f6ea0 100644 --- a/pkg/controller/mocks/Interface.go +++ b/pkg/controller/mocks/Interface.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.52.2. DO NOT EDIT. package mocks @@ -9,6 +9,8 @@ import ( ecs "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + eflo "github.com/aliyun/alibaba-cloud-sdk-go/services/eflo" + mock "github.com/stretchr/testify/mock" netip "net/netip" @@ -60,6 +62,43 @@ func (_m *Interface) AssignIpv6Addresses(ctx context.Context, opts ...client.Ass return r0, r1 } +// AssignIpv6AddressesV2 provides a mock function with given fields: ctx, opts +func (_m *Interface) AssignIpv6AddressesV2(ctx context.Context, opts ...client.AssignIPv6AddressesOption) ([]client.IPSet, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for AssignIpv6AddressesV2") + } + + var r0 []client.IPSet + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, ...client.AssignIPv6AddressesOption) ([]client.IPSet, error)); ok { + return rf(ctx, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, ...client.AssignIPv6AddressesOption) []client.IPSet); ok { + r0 = rf(ctx, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]client.IPSet) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, ...client.AssignIPv6AddressesOption) error); ok { + r1 = rf(ctx, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // AssignPrivateIPAddress provides a mock function with given fields: ctx, opts func (_m *Interface) AssignPrivateIPAddress(ctx context.Context, opts ...client.AssignPrivateIPAddressOption) ([]netip.Addr, error) { _va := make([]interface{}, len(opts)) @@ -97,6 +136,43 @@ func (_m *Interface) AssignPrivateIPAddress(ctx context.Context, opts ...client. return r0, r1 } +// AssignPrivateIPAddressV2 provides a mock function with given fields: ctx, opts +func (_m *Interface) AssignPrivateIPAddressV2(ctx context.Context, opts ...client.AssignPrivateIPAddressOption) ([]client.IPSet, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for AssignPrivateIPAddressV2") + } + + var r0 []client.IPSet + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, ...client.AssignPrivateIPAddressOption) ([]client.IPSet, error)); ok { + return rf(ctx, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, ...client.AssignPrivateIPAddressOption) []client.IPSet); ok { + r0 = rf(ctx, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]client.IPSet) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, ...client.AssignPrivateIPAddressOption) error); ok { + r1 = rf(ctx, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // AttachNetworkInterface provides a mock function with given fields: ctx, eniID, instanceID, trunkENIID func (_m *Interface) AttachNetworkInterface(ctx context.Context, eniID string, instanceID string, trunkENIID string) error { ret := _m.Called(ctx, eniID, instanceID, trunkENIID) @@ -152,6 +228,43 @@ func (_m *Interface) CreateNetworkInterface(ctx context.Context, opts ...client. return r0, r1 } +// CreateNetworkInterfaceV2 provides a mock function with given fields: ctx, opts +func (_m *Interface) CreateNetworkInterfaceV2(ctx context.Context, opts ...client.CreateNetworkInterfaceOption) (*client.NetworkInterface, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for CreateNetworkInterfaceV2") + } + + var r0 *client.NetworkInterface + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, ...client.CreateNetworkInterfaceOption) (*client.NetworkInterface, error)); ok { + return rf(ctx, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, ...client.CreateNetworkInterfaceOption) *client.NetworkInterface); ok { + r0 = rf(ctx, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*client.NetworkInterface) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, ...client.CreateNetworkInterfaceOption) error); ok { + r1 = rf(ctx, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // DeleteNetworkInterface provides a mock function with given fields: ctx, eniID func (_m *Interface) DeleteNetworkInterface(ctx context.Context, eniID string) error { ret := _m.Called(ctx, eniID) @@ -170,6 +283,24 @@ func (_m *Interface) DeleteNetworkInterface(ctx context.Context, eniID string) e return r0 } +// DeleteNetworkInterfaceV2 provides a mock function with given fields: ctx, eniID +func (_m *Interface) DeleteNetworkInterfaceV2(ctx context.Context, eniID string) error { + ret := _m.Called(ctx, eniID) + + if len(ret) == 0 { + panic("no return value specified for DeleteNetworkInterfaceV2") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, eniID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // DescribeInstanceTypes provides a mock function with given fields: ctx, types func (_m *Interface) DescribeInstanceTypes(ctx context.Context, types []string) ([]ecs.InstanceType, error) { ret := _m.Called(ctx, types) @@ -230,6 +361,43 @@ func (_m *Interface) DescribeNetworkInterface(ctx context.Context, vpcID string, return r0, r1 } +// DescribeNetworkInterfaceV2 provides a mock function with given fields: ctx, opts +func (_m *Interface) DescribeNetworkInterfaceV2(ctx context.Context, opts ...client.DescribeNetworkInterfaceOption) ([]*client.NetworkInterface, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for DescribeNetworkInterfaceV2") + } + + var r0 []*client.NetworkInterface + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, ...client.DescribeNetworkInterfaceOption) ([]*client.NetworkInterface, error)); ok { + return rf(ctx, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, ...client.DescribeNetworkInterfaceOption) []*client.NetworkInterface); ok { + r0 = rf(ctx, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*client.NetworkInterface) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, ...client.DescribeNetworkInterfaceOption) error); ok { + r1 = rf(ctx, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // DescribeVSwitchByID provides a mock function with given fields: ctx, vSwitchID func (_m *Interface) DescribeVSwitchByID(ctx context.Context, vSwitchID string) (*vpc.VSwitch, error) { ret := _m.Called(ctx, vSwitchID) @@ -278,22 +446,34 @@ func (_m *Interface) DetachNetworkInterface(ctx context.Context, eniID string, i return r0 } -// ModifyNetworkInterfaceAttribute provides a mock function with given fields: ctx, eniID, securityGroupIDs -func (_m *Interface) ModifyNetworkInterfaceAttribute(ctx context.Context, eniID string, securityGroupIDs []string) error { - ret := _m.Called(ctx, eniID, securityGroupIDs) +// GetNodeInfoForPod provides a mock function with given fields: ctx, nodeID +func (_m *Interface) GetNodeInfoForPod(ctx context.Context, nodeID string) (*eflo.Content, error) { + ret := _m.Called(ctx, nodeID) if len(ret) == 0 { - panic("no return value specified for ModifyNetworkInterfaceAttribute") + panic("no return value specified for GetNodeInfoForPod") } - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, []string) error); ok { - r0 = rf(ctx, eniID, securityGroupIDs) + var r0 *eflo.Content + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*eflo.Content, error)); ok { + return rf(ctx, nodeID) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *eflo.Content); ok { + r0 = rf(ctx, nodeID) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*eflo.Content) + } } - return r0 + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, nodeID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // UnAssignIpv6Addresses provides a mock function with given fields: ctx, eniID, ips @@ -314,6 +494,24 @@ func (_m *Interface) UnAssignIpv6Addresses(ctx context.Context, eniID string, ip return r0 } +// UnAssignIpv6AddressesV2 provides a mock function with given fields: ctx, eniID, ips +func (_m *Interface) UnAssignIpv6AddressesV2(ctx context.Context, eniID string, ips []client.IPSet) error { + ret := _m.Called(ctx, eniID, ips) + + if len(ret) == 0 { + panic("no return value specified for UnAssignIpv6AddressesV2") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, []client.IPSet) error); ok { + r0 = rf(ctx, eniID, ips) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // UnAssignPrivateIPAddresses provides a mock function with given fields: ctx, eniID, ips func (_m *Interface) UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []netip.Addr) error { ret := _m.Called(ctx, eniID, ips) @@ -332,6 +530,24 @@ func (_m *Interface) UnAssignPrivateIPAddresses(ctx context.Context, eniID strin return r0 } +// UnAssignPrivateIPAddressesV2 provides a mock function with given fields: ctx, eniID, ips +func (_m *Interface) UnAssignPrivateIPAddressesV2(ctx context.Context, eniID string, ips []client.IPSet) error { + ret := _m.Called(ctx, eniID, ips) + + if len(ret) == 0 { + panic("no return value specified for UnAssignPrivateIPAddressesV2") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, []client.IPSet) error); ok { + r0 = rf(ctx, eniID, ips) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // WaitForNetworkInterface provides a mock function with given fields: ctx, eniID, status, backoff, ignoreNotExist func (_m *Interface) WaitForNetworkInterface(ctx context.Context, eniID string, status string, backoff wait.Backoff, ignoreNotExist bool) (*client.NetworkInterface, error) { ret := _m.Called(ctx, eniID, status, backoff, ignoreNotExist) @@ -362,6 +578,36 @@ func (_m *Interface) WaitForNetworkInterface(ctx context.Context, eniID string, return r0, r1 } +// WaitForNetworkInterfaceV2 provides a mock function with given fields: ctx, eniID, status, backoff, ignoreNotExist +func (_m *Interface) WaitForNetworkInterfaceV2(ctx context.Context, eniID string, status string, backoff wait.Backoff, ignoreNotExist bool) (*client.NetworkInterface, error) { + ret := _m.Called(ctx, eniID, status, backoff, ignoreNotExist) + + if len(ret) == 0 { + panic("no return value specified for WaitForNetworkInterfaceV2") + } + + var r0 *client.NetworkInterface + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, wait.Backoff, bool) (*client.NetworkInterface, error)); ok { + return rf(ctx, eniID, status, backoff, ignoreNotExist) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, wait.Backoff, bool) *client.NetworkInterface); ok { + r0 = rf(ctx, eniID, status, backoff, ignoreNotExist) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*client.NetworkInterface) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, wait.Backoff, bool) error); ok { + r1 = rf(ctx, eniID, status, backoff, ignoreNotExist) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // NewInterface creates a new instance of Interface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewInterface(t interface { diff --git a/pkg/controller/multi-ip/node/pool.go b/pkg/controller/multi-ip/node/pool.go index 7fe4e3c1..ab9d22b2 100644 --- a/pkg/controller/multi-ip/node/pool.go +++ b/pkg/controller/multi-ip/node/pool.go @@ -1049,7 +1049,7 @@ func (n *ReconcileNode) handleStatus(ctx context.Context, node *networkv1beta1.N log.Error(err, "run gc failed") continue } - _, err = n.aliyun.WaitForNetworkInterface(ctx, eni.ID, aliyunClient.ENIStatusAvailable, backoff.Backoff(backoff.WaitENIStatus), true) + _, err = n.aliyun.WaitForNetworkInterfaceV2(ctx, eni.ID, aliyunClient.ENIStatusAvailable, backoff.Backoff(backoff.WaitENIStatus), true) if err != nil { if !errors.Is(err, apiErr.ErrNotFound) { log.Error(err, "run gc failed") @@ -1210,7 +1210,7 @@ func (n *ReconcileNode) createENI(ctx context.Context, node *networkv1beta1.Node VSwitchID: vsw.ID, SecurityGroupIDs: node.Spec.ENISpec.SecurityGroupIDs, ResourceGroupID: node.Spec.ENISpec.ResourceGroupID, - Tags: node.Spec.ENISpec.Tag, + Tags: tags, IPCount: opt.addIPv4N, IPv6Count: opt.addIPv6N, diff --git a/pkg/controller/multi-ip/node/pool_test.go b/pkg/controller/multi-ip/node/pool_test.go index e4f2c2c8..ffdf6130 100644 --- a/pkg/controller/multi-ip/node/pool_test.go +++ b/pkg/controller/multi-ip/node/pool_test.go @@ -394,7 +394,7 @@ func TestReconcileNodeSyncWithAPI(t *testing.T) { CidrBlock: "192.168.0.0/16", Ipv6CidrBlock: "fd00::/64", }, nil).Maybe() - openAPI.On("DescribeNetworkInterface", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]*aliyunClient.NetworkInterface{ + openAPI.On("DescribeNetworkInterfaceV2", mock.Anything, mock.Anything).Return([]*aliyunClient.NetworkInterface{ { Status: "InUse", MacAddress: "", @@ -876,13 +876,21 @@ func TestReconcileNode_assignIP(t *testing.T) { fields: fields{ aliyun: func() register.Interface { openAPI := mocks.NewInterface(t) - openAPI.On("AssignPrivateIPAddress", mock.Anything, mock.Anything).Return([]netip.Addr{ - netip.MustParseAddr("192.168.0.1"), - netip.MustParseAddr("192.168.0.2"), + openAPI.On("AssignPrivateIPAddressV2", mock.Anything, mock.Anything).Return([]aliyunClient.IPSet{ + { + IPAddress: netip.MustParseAddr("192.168.0.1").String(), + }, + { + IPAddress: netip.MustParseAddr("192.168.0.2").String(), + }, }, nil) - openAPI.On("AssignIpv6Addresses", mock.Anything, mock.Anything).Return([]netip.Addr{ - netip.MustParseAddr("fd00::1"), - netip.MustParseAddr("fd00::2"), + openAPI.On("AssignIpv6AddressesV2", mock.Anything, mock.Anything).Return([]aliyunClient.IPSet{ + { + IPAddress: netip.MustParseAddr("fd00::1").String(), + }, + { + IPAddress: netip.MustParseAddr("fd00::2").String(), + }, }, nil) return openAPI }(), @@ -1007,7 +1015,7 @@ func TestReconcileNode_createENI(t *testing.T) { fields: fields{ aliyun: func() register.Interface { openAPI := mocks.NewInterface(t) - openAPI.On("CreateNetworkInterface", mock.Anything, mock.Anything, mock.Anything).Return(&aliyunClient.NetworkInterface{ + openAPI.On("CreateNetworkInterfaceV2", mock.Anything, mock.Anything, mock.Anything).Return(&aliyunClient.NetworkInterface{ Status: "Available", MacAddress: "", NetworkInterfaceID: "eni-1", @@ -1040,7 +1048,7 @@ func TestReconcileNode_createENI(t *testing.T) { NetworkInterfaceTrafficMode: "", }, nil) openAPI.On("AttachNetworkInterface", mock.Anything, "eni-1", mock.Anything, "").Return(nil) - openAPI.On("WaitForNetworkInterface", mock.Anything, "eni-1", mock.Anything, mock.Anything, mock.Anything).Return(&aliyunClient.NetworkInterface{ + openAPI.On("WaitForNetworkInterfaceV2", mock.Anything, "eni-1", mock.Anything, mock.Anything, mock.Anything).Return(&aliyunClient.NetworkInterface{ Status: "InUse", MacAddress: "", NetworkInterfaceID: "eni-1", @@ -1133,7 +1141,7 @@ func TestReconcileNode_createENI(t *testing.T) { fields: fields{ aliyun: func() register.Interface { openAPI := mocks.NewInterface(t) - openAPI.On("CreateNetworkInterface", mock.Anything, mock.Anything, mock.Anything).Return(&aliyunClient.NetworkInterface{ + openAPI.On("CreateNetworkInterfaceV2", mock.Anything, mock.Anything, mock.Anything).Return(&aliyunClient.NetworkInterface{ Status: "Available", MacAddress: "", NetworkInterfaceID: "eni-1", @@ -1166,8 +1174,8 @@ func TestReconcileNode_createENI(t *testing.T) { NetworkInterfaceTrafficMode: "", }, nil) openAPI.On("AttachNetworkInterface", mock.Anything, "eni-1", mock.Anything, "").Return(nil) - openAPI.On("WaitForNetworkInterface", mock.Anything, "eni-1", mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("time out")) - openAPI.On("DeleteNetworkInterface", mock.Anything, "eni-1").Return(fmt.Errorf("eni already attached")) + openAPI.On("WaitForNetworkInterfaceV2", mock.Anything, "eni-1", mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("time out")) + openAPI.On("DeleteNetworkInterfaceV2", mock.Anything, "eni-1").Return(fmt.Errorf("eni already attached")) return openAPI }(), vswpool: func() *vswpool.SwitchPool { @@ -1577,7 +1585,7 @@ func Test_assignEniWithOptions(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assignEniWithOptions(tt.args.node, tt.args.toAdd, tt.args.options, tt.args.filterFunc) + assignEniWithOptions(context.Background(), tt.args.node, tt.args.toAdd, tt.args.options, tt.args.filterFunc) tt.checkResult(t, tt.args.options) }) @@ -1604,8 +1612,8 @@ func TestReconcileNode_handleStatus(t *testing.T) { fields: fields{ aliyun: func() register.Interface { openAPI := mocks.NewInterface(t) - openAPI.On("UnAssignPrivateIPAddresses", mock.Anything, "eni-1", mock.Anything).Return(nil) - openAPI.On("UnAssignIpv6Addresses", mock.Anything, "eni-1", mock.Anything).Return(nil) + openAPI.On("UnAssignPrivateIPAddressesV2", mock.Anything, "eni-1", mock.Anything).Return(nil) + openAPI.On("UnAssignIpv6AddressesV2", mock.Anything, "eni-1", mock.Anything).Return(nil) return openAPI }(), }, @@ -1659,7 +1667,7 @@ func TestReconcileNode_handleStatus(t *testing.T) { aliyun: func() register.Interface { openAPI := mocks.NewInterface(t) openAPI.On("DetachNetworkInterface", mock.Anything, "eni-1", mock.Anything, mock.Anything).Return(nil) - openAPI.On("WaitForNetworkInterface", mock.Anything, "eni-1", mock.Anything, mock.Anything, mock.Anything).Return(&aliyunClient.NetworkInterface{ + openAPI.On("WaitForNetworkInterfaceV2", mock.Anything, "eni-1", mock.Anything, mock.Anything, mock.Anything).Return(&aliyunClient.NetworkInterface{ Status: "Available", MacAddress: "", NetworkInterfaceID: "eni-1", @@ -1694,7 +1702,7 @@ func TestReconcileNode_handleStatus(t *testing.T) { DeviceIndex: 0, CreationTime: "", }, nil) - openAPI.On("DeleteNetworkInterface", mock.Anything, "eni-1").Return(nil) + openAPI.On("DeleteNetworkInterfaceV2", mock.Anything, "eni-1").Return(nil) return openAPI }(), }, diff --git a/pkg/controller/node/node.go b/pkg/controller/node/node.go index beb9e238..932f2a87 100644 --- a/pkg/controller/node/node.go +++ b/pkg/controller/node/node.go @@ -12,6 +12,7 @@ import ( k8sErr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -29,6 +30,8 @@ import ( register "github.com/AliyunContainerService/terway/pkg/controller" "github.com/AliyunContainerService/terway/pkg/controller/common" "github.com/AliyunContainerService/terway/pkg/controller/multi-ip/node" + "github.com/AliyunContainerService/terway/pkg/feature" + "github.com/AliyunContainerService/terway/pkg/utils" "github.com/AliyunContainerService/terway/types" "github.com/AliyunContainerService/terway/types/controlplane" ) @@ -63,14 +66,17 @@ func init() { return log }, }). - For(&corev1.Node{}, builder.WithPredicates(&predicateForNodeEvent{})). + For(&corev1.Node{}, builder.WithPredicates(&predicateForNodeEvent{ + supportEFLO: utilfeature.DefaultMutableFeatureGate.Enabled(feature.EFLO), + })). Watches(&networkv1beta1.Node{}, &handler.EnqueueRequestForObject{}). Watches(&networkv1beta1.NodeRuntime{}, &handler.EnqueueRequestForObject{}). Complete(&ReconcileNode{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - record: mgr.GetEventRecorderFor(ControllerName), - aliyun: ctrlCtx.AliyunClient, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + record: mgr.GetEventRecorderFor(ControllerName), + aliyun: ctrlCtx.AliyunClient, + supportEFLO: utilfeature.DefaultMutableFeatureGate.Enabled(feature.EFLO), }) }, false) } @@ -83,6 +89,8 @@ type ReconcileNode struct { aliyun register.Interface record record.EventRecorder + + supportEFLO bool } func (r *ReconcileNode) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { @@ -96,7 +104,7 @@ func (r *ReconcileNode) Reconcile(ctx context.Context, request reconcile.Request } return reconcile.Result{}, err } - if !predicateNode(k8sNode) { + if !predicateNode(k8sNode, r.supportEFLO) { return reconcile.Result{}, nil } if !k8sNode.DeletionTimestamp.IsZero() { @@ -120,6 +128,9 @@ func (r *ReconcileNode) Reconcile(ctx context.Context, request reconcile.Request return reconcile.Result{}, nil } + if utils.ISLinJunNode(k8sNode.Labels) { + return r.handleEFLO(ctx, k8sNode, node) + } // create or update the crdNode err = r.createOrUpdate(ctx, k8sNode, node) if err != nil { @@ -339,3 +350,28 @@ func (r *ReconcileNode) patchNodeRes(ctx context.Context, k8sNode *corev1.Node, return nil } + +func (r *ReconcileNode) handleEFLO(ctx context.Context, k8sNode *corev1.Node, node *networkv1beta1.Node) (reconcile.Result, error) { + if node.Labels == nil { + node.Labels = make(map[string]string) + } + + if node.Spec.NodeMetadata.InstanceID != "" && node.Spec.NodeCap.Adapters == 0 { + resp, err := r.aliyun.GetNodeInfoForPod(ctx, node.Spec.NodeMetadata.InstanceID) + if err != nil { + return reconcile.Result{}, err + } + node.Spec.NodeCap.Adapters = resp.LeniQuota + node.Spec.NodeCap.TotalAdapters = resp.LeniQuota + node.Spec.NodeCap.IPv4PerAdapter = resp.LniSipQuota + } + + update := node.DeepCopy() + _, err := controllerutil.CreateOrPatch(ctx, r.client, update, func() error { + update.Status = node.Status + update.Spec = node.Spec + update.Labels = node.Labels + return nil + }) + return reconcile.Result{}, err +} diff --git a/pkg/controller/node/node_controller_test.go b/pkg/controller/node/node_controller_test.go index a473d11b..9d79f23f 100644 --- a/pkg/controller/node/node_controller_test.go +++ b/pkg/controller/node/node_controller_test.go @@ -128,19 +128,24 @@ var _ = Describe("Node Controller", func() { Name: resourceName, }, Spec: networkv1beta1.NodeSpec{ - NodeMetadata: networkv1beta1.NodeMetadata{}, - NodeCap: networkv1beta1.NodeCap{}, + NodeMetadata: networkv1beta1.NodeMetadata{ + RegionID: "foo", + InstanceType: "foo", + InstanceID: "foo", + ZoneID: "foo", + }, + NodeCap: networkv1beta1.NodeCap{}, ENISpec: &networkv1beta1.ENISpec{ Tag: nil, TagFilter: nil, - VSwitchOptions: nil, - SecurityGroupIDs: nil, + VSwitchOptions: []string{"foo"}, + SecurityGroupIDs: []string{"foo"}, ResourceGroupID: "", EnableIPv4: true, EnableIPv6: false, EnableERDMA: false, EnableTrunk: true, - VSwitchSelectPolicy: "", + VSwitchSelectPolicy: "ordered", }, Pool: nil, Flavor: []networkv1beta1.Flavor{ @@ -169,6 +174,12 @@ var _ = Describe("Node Controller", func() { NetworkInterfaceTrafficMode: networkv1beta1.NetworkInterfaceTrafficModeStandard, SecurityGroupIDs: []string{"ff"}, Status: "InUse", + IPv4: map[string]*networkv1beta1.IP{ + "192.168.0.1": { + IP: "192.168.0.1", + Status: "Valid", + }, + }, }, }, } @@ -201,19 +212,24 @@ var _ = Describe("Node Controller", func() { Name: resourceName, }, Spec: networkv1beta1.NodeSpec{ - NodeMetadata: networkv1beta1.NodeMetadata{}, - NodeCap: networkv1beta1.NodeCap{}, + NodeMetadata: networkv1beta1.NodeMetadata{ + RegionID: "foo", + InstanceType: "foo", + InstanceID: "foo", + ZoneID: "foo", + }, + NodeCap: networkv1beta1.NodeCap{}, ENISpec: &networkv1beta1.ENISpec{ Tag: nil, TagFilter: nil, - VSwitchOptions: nil, - SecurityGroupIDs: nil, + VSwitchOptions: []string{"foo"}, + SecurityGroupIDs: []string{"foo"}, ResourceGroupID: "", EnableIPv4: true, EnableIPv6: false, EnableERDMA: false, EnableTrunk: true, - VSwitchSelectPolicy: "", + VSwitchSelectPolicy: "ordered", }, Pool: nil, Flavor: []networkv1beta1.Flavor{ @@ -247,7 +263,12 @@ var _ = Describe("Node Controller", func() { NetworkInterfaceTrafficMode: networkv1beta1.NetworkInterfaceTrafficModeStandard, SecurityGroupIDs: []string{"ff"}, Status: "InUse", - }, + IPv4: map[string]*networkv1beta1.IP{ + "192.168.0.1": { + IP: "192.168.0.1", + Status: "Valid", + }, + }}, }, } return nil @@ -369,19 +390,24 @@ var _ = Describe("Node Exclusive eni node", func() { Name: resourceName, }, Spec: networkv1beta1.NodeSpec{ - NodeMetadata: networkv1beta1.NodeMetadata{}, - NodeCap: networkv1beta1.NodeCap{}, + NodeMetadata: networkv1beta1.NodeMetadata{ + RegionID: "foo", + InstanceType: "foo", + InstanceID: "foo", + ZoneID: "foo", + }, + NodeCap: networkv1beta1.NodeCap{}, ENISpec: &networkv1beta1.ENISpec{ Tag: nil, TagFilter: nil, - VSwitchOptions: nil, - SecurityGroupIDs: nil, + VSwitchOptions: []string{"foo"}, + SecurityGroupIDs: []string{"foo"}, ResourceGroupID: "", EnableIPv4: true, EnableIPv6: false, EnableERDMA: false, EnableTrunk: true, - VSwitchSelectPolicy: "", + VSwitchSelectPolicy: "ordered", }, Pool: nil, Flavor: []networkv1beta1.Flavor{ @@ -410,6 +436,12 @@ var _ = Describe("Node Exclusive eni node", func() { NetworkInterfaceTrafficMode: networkv1beta1.NetworkInterfaceTrafficModeStandard, SecurityGroupIDs: []string{"ff"}, Status: "InUse", + IPv4: map[string]*networkv1beta1.IP{ + "192.168.0.1": { + IP: "192.168.0.1", + Status: "Valid", + }, + }, }, }, } diff --git a/pkg/controller/node/predict.go b/pkg/controller/node/predict.go index 27d397c7..bb4832c1 100644 --- a/pkg/controller/node/predict.go +++ b/pkg/controller/node/predict.go @@ -28,51 +28,55 @@ import ( type predicateForNodeEvent struct { predicate.Funcs + + supportEFLO bool } // Create returns true if the Create event should be processed func (p *predicateForNodeEvent) Create(e event.CreateEvent) bool { - return predicateNode(e.Object) + return predicateNode(e.Object, p.supportEFLO) } // Delete returns true if the Delete event should be processed func (p *predicateForNodeEvent) Delete(e event.DeleteEvent) bool { - return predicateNode(e.Object) + return predicateNode(e.Object, p.supportEFLO) } // Update returns true if the Update event should be processed func (p *predicateForNodeEvent) Update(e event.UpdateEvent) bool { - return predicateNode(e.ObjectNew) + return predicateNode(e.ObjectNew, p.supportEFLO) } // Generic returns true if the Generic event should be processed func (p *predicateForNodeEvent) Generic(e event.GenericEvent) bool { - return predicateNode(e.Object) + return predicateNode(e.Object, p.supportEFLO) } -func predicateNode(o client.Object) bool { +func predicateNode(o client.Object, supportEFLO bool) bool { node, ok := o.(*corev1.Node) if !ok { return false } - if node.Labels[corev1.LabelTopologyRegion] == "" { - return false + if !supportEFLO { + if node.Labels[corev1.LabelTopologyRegion] == "" { + return false + } } if types.IgnoredByTerway(node.Labels) { return false } - return isECSNode(node) -} - -func isECSNode(node *corev1.Node) bool { - if utils.ISLinJunNode(node.Labels) { - return false + if !supportEFLO { + if utils.ISLinJunNode(node.Labels) { + return false + } } + if utils.ISVKNode(node) { return false } + return true } diff --git a/pkg/controller/node/predict_test.go b/pkg/controller/node/predict_test.go index 6a7e9eb6..050a15f8 100644 --- a/pkg/controller/node/predict_test.go +++ b/pkg/controller/node/predict_test.go @@ -3,121 +3,96 @@ package node import ( "testing" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) -func Test_isECSNode(t *testing.T) { - type args struct { - node *corev1.Node - } +func TestPredicateNode(t *testing.T) { tests := []struct { - name string - args args - want bool + name string + node *corev1.Node + supportEFLO bool + expected bool }{ { - name: "normal node", - args: args{ - node: &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{}, + name: "SupportEFLOFalseAndNoRegionLabel", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "other-label": "value", + }, }, }, - want: true, + supportEFLO: false, + expected: false, }, { - name: "vk node", - args: args{ - node: &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "type": "virtual-kubelet", - }, + name: "IgnoredByTerway", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "topology.kubernetes.io/region": "region", + "k8s.aliyun.com/ignore-by-terway": "true", }, }, }, - want: false, + expected: false, }, { - name: "linjun node", - args: args{ - node: &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "alibabacloud.com/lingjun-worker": "true", - }, + name: "Ignore Lunjun worker", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "topology.kubernetes.io/region": "region", + "alibabacloud.com/lingjun-worker": "true", }, }, }, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isECSNode(tt.args.node); got != tt.want { - t.Errorf("isECSNode() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_predicateNode(t *testing.T) { - type args struct { - o client.Object - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "non node", - args: args{ - o: &corev1.Pod{}, - }, - want: false, + supportEFLO: false, + expected: false, }, { - name: "empty node", - args: args{ - o: &corev1.Node{}, + name: "Lunjun worker", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "alibabacloud.com/lingjun-worker": "true", + }, + }, }, - want: false, + supportEFLO: true, + expected: true, }, { - name: "normal node", - args: args{ - o: &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "topology.kubernetes.io/region": "cn-hangzhou", - }, + name: "VKNode", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "topology.kubernetes.io/region": "region", + "type": "virtual-kubelet", }, }, }, - want: true, + expected: false, }, { - name: "normal node but has exclude rule", - args: args{ - o: &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "topology.kubernetes.io/region": "cn-hangzhou", - "k8s.aliyun.com/ignore-by-terway": "true", - }, + name: "AllConditionsSatisfied", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "topology.kubernetes.io/region": "region", }, }, }, - want: false, + expected: true, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := predicateNode(tt.args.o); got != tt.want { - t.Errorf("predicateNode() = %v, want %v", got, tt.want) - } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := predicateNode(test.node, test.supportEFLO) + assert.Equal(t, test.expected, result) }) } } diff --git a/pkg/controller/register.go b/pkg/controller/register.go index 94420e48..010629a7 100644 --- a/pkg/controller/register.go +++ b/pkg/controller/register.go @@ -37,6 +37,7 @@ type Interface interface { aliyunClient.VPC aliyunClient.ECS aliyunClient.ENI + aliyunClient.EFLO } type ControllerCtx struct { diff --git a/pkg/eni/local.go b/pkg/eni/local.go index bb2b8536..cbed6471 100644 --- a/pkg/eni/local.go +++ b/pkg/eni/local.go @@ -155,7 +155,7 @@ type Local struct { factory factory.Factory } -func NewLocal(eni *daemon.ENI, eniType string, factory factory.Factory, poolConfig *types.PoolConfig) *Local { +func NewLocal(eni *daemon.ENI, eniType string, factory factory.Factory, poolConfig *daemon.PoolConfig) *Local { l := &Local{ eni: eni, batchSize: poolConfig.BatchSize, diff --git a/pkg/eni/local_test.go b/pkg/eni/local_test.go index d8cb34e6..1a71a10f 100644 --- a/pkg/eni/local_test.go +++ b/pkg/eni/local_test.go @@ -21,7 +21,7 @@ import ( "github.com/AliyunContainerService/terway/types/daemon" ) -func NewLocalTest(eni *daemon.ENI, factory factory.Factory, poolConfig *types.PoolConfig, eniType string) *Local { +func NewLocalTest(eni *daemon.ENI, factory factory.Factory, poolConfig *daemon.PoolConfig, eniType string) *Local { l := &Local{ eni: eni, batchSize: poolConfig.BatchSize, @@ -43,7 +43,7 @@ func NewLocalTest(eni *daemon.ENI, factory factory.Factory, poolConfig *types.Po } func TestLocal_Release_ValidIPv4(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{}, "") request := &LocalIPResource{ ENI: daemon.ENI{ID: "eni-1"}, IP: types.IPSet2{IPv4: netip.MustParseAddr("192.0.2.1")}, @@ -58,7 +58,7 @@ func TestLocal_Release_ValidIPv4(t *testing.T) { } func TestLocal_Release_ValidIPv6(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{}, "") request := &LocalIPResource{ ENI: daemon.ENI{ID: "eni-1"}, IP: types.IPSet2{IPv6: netip.MustParseAddr("fd00:46dd:e::1")}, @@ -73,7 +73,7 @@ func TestLocal_Release_ValidIPv6(t *testing.T) { } func TestLocal_Release_NilENI(t *testing.T) { - local := NewLocalTest(nil, nil, &types.PoolConfig{}, "") + local := NewLocalTest(nil, nil, &daemon.PoolConfig{}, "") request := &LocalIPResource{ ENI: daemon.ENI{ID: "eni-1"}, IP: types.IPSet2{IPv4: netip.MustParseAddr("192.0.2.1")}, @@ -85,7 +85,7 @@ func TestLocal_Release_NilENI(t *testing.T) { } func TestLocal_Release_DifferentENIID(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{}, "") request := &LocalIPResource{ ENI: daemon.ENI{ID: "eni-2"}, IP: types.IPSet2{IPv4: netip.MustParseAddr("192.0.2.1")}, @@ -97,7 +97,7 @@ func TestLocal_Release_DifferentENIID(t *testing.T) { } func TestLocal_Release_ValidIPv4_ReleaseIPv6(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{}, "") request := &LocalIPResource{ ENI: daemon.ENI{ID: "eni-1"}, IP: types.IPSet2{IPv4: netip.MustParseAddr("192.0.2.1"), IPv6: netip.MustParseAddr("fd00:46dd:e::1")}, @@ -121,7 +121,7 @@ func TestLocal_Release_ValidIPv4_ReleaseIPv6(t *testing.T) { } func TestLocal_AllocWorker_EnableIPv4(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{ EnableIPv4: true, }, "") cni := &daemon.CNI{PodID: "pod-1"} @@ -146,7 +146,7 @@ func TestLocal_AllocWorker_EnableIPv4(t *testing.T) { } func TestLocal_AllocWorker_EnableIPv6(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{ EnableIPv6: true, }, "") cni := &daemon.CNI{PodID: "pod-1"} @@ -171,7 +171,7 @@ func TestLocal_AllocWorker_EnableIPv6(t *testing.T) { } func TestLocal_AllocWorker_ParentCancelContext(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{ EnableIPv4: true, }, "") cni := &daemon.CNI{PodID: "pod-1"} @@ -187,7 +187,7 @@ func TestLocal_AllocWorker_ParentCancelContext(t *testing.T) { } func TestLocal_AllocWorker_UpdateCache(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{ EnableIPv4: true, }, "") cni := &daemon.CNI{PodID: "pod-1"} @@ -206,7 +206,7 @@ func TestLocal_AllocWorker_UpdateCache(t *testing.T) { } func TestLocal_Dispose(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{}, "") local.status = statusInUse local.ipv4.Add(NewValidIP(netip.MustParseAddr("192.0.2.1"), false)) local.ipv4[netip.MustParseAddr("192.0.2.1")].Allocate("pod-1") @@ -222,7 +222,7 @@ func TestLocal_Dispose(t *testing.T) { } func TestLocal_DisposeWholeENI(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{}, "") local.status = statusInUse local.ipv4.Add(NewValidIP(netip.MustParseAddr("192.0.2.1"), true)) local.ipv6.Add(NewValidIP(netip.MustParseAddr("fd00:46dd:e::1"), false)) @@ -234,7 +234,7 @@ func TestLocal_DisposeWholeENI(t *testing.T) { } func TestLocal_Allocate_NoCache(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{MaxIPPerENI: 2, EnableIPv4: true}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{MaxIPPerENI: 2, EnableIPv4: true}, "") request := NewLocalIPRequest() request.NoCache = true @@ -249,7 +249,7 @@ func TestLocal_Allocate_NoCache(t *testing.T) { } func TestLocal_DisposeFailWhenAllocatingIsNotEmpty(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{}, "") local.status = statusInUse local.ipv4.Add(NewValidIP(netip.MustParseAddr("192.0.2.1"), true)) local.ipv6.Add(NewValidIP(netip.MustParseAddr("fd00:46dd:e::1"), false)) @@ -262,7 +262,7 @@ func TestLocal_DisposeFailWhenAllocatingIsNotEmpty(t *testing.T) { } func TestLocal_Allocate_NoCache_AllocSuccess(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{ MaxIPPerENI: 10, EnableIPv4: true, EnableIPv6: true}, "") request := NewLocalIPRequest() @@ -278,7 +278,7 @@ func TestLocal_Allocate_NoCache_AllocSuccess(t *testing.T) { } func TestLocal_DisposeWholeERDMA(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}, "erdma") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{}, "erdma") local.status = statusInUse local.ipv4.Add(NewValidIP(netip.MustParseAddr("192.0.2.1"), false)) @@ -289,7 +289,7 @@ func TestLocal_DisposeWholeERDMA(t *testing.T) { } func TestLocal_Allocate_ERDMA(t *testing.T) { - localErdma := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{MaxIPPerENI: 2, EnableIPv4: true}, "erdma") + localErdma := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{MaxIPPerENI: 2, EnableIPv4: true}, "erdma") request := NewLocalIPRequest() request.NoCache = true @@ -311,7 +311,7 @@ func TestLocal_Allocate_ERDMA(t *testing.T) { assert.Equal(t, 1, len(resp)) assert.NotEqual(t, ResourceTypeMismatch, resp[0].Condition) - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{MaxIPPerENI: 2, EnableIPv4: true}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{MaxIPPerENI: 2, EnableIPv4: true}, "") local.ipv4.Add(NewValidIP(netip.MustParseAddr("192.0.2.1"), false)) local.ipv4.Add(NewValidIP(netip.MustParseAddr("192.0.2.2"), false)) @@ -332,7 +332,7 @@ func TestLocal_Allocate_ERDMA(t *testing.T) { } func TestLocal_Allocate_Inhibit(t *testing.T) { - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{MaxIPPerENI: 2, EnableIPv4: true}, "") + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &daemon.PoolConfig{MaxIPPerENI: 2, EnableIPv4: true}, "") request := NewLocalIPRequest() cni := &daemon.CNI{PodID: "pod-1"} @@ -564,7 +564,7 @@ func TestAllocFromFactory(t *testing.T) { f.On("AssignNIPv4", "eni-1", 1, "").Return(nil, nil).Maybe() f.On("AssignNIPv6", "eni-1", 1, "").Return(nil, nil).Maybe() - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &daemon.PoolConfig{ EnableIPv4: true, EnableIPv6: true, BatchSize: 10, @@ -612,7 +612,7 @@ func Test_factoryDisposeWorker_unAssignIP(t *testing.T) { f.On("UnAssignNIPv4", "eni-1", []netip.Addr{netip.MustParseAddr("192.0.2.1")}, mock.Anything).Return(nil).Once() f.On("UnAssignNIPv6", "eni-1", []netip.Addr{netip.MustParseAddr("fd00::1")}, mock.Anything).Return(nil).Once() - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &daemon.PoolConfig{ EnableIPv4: true, EnableIPv6: true, BatchSize: 10, @@ -664,7 +664,7 @@ func Test_factoryDisposeWorker_releaseIP(t *testing.T) { // even we have two jobs ,we only get one ip f.On("DeleteNetworkInterface", "eni-1").Return(nil).Once() - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &daemon.PoolConfig{ EnableIPv4: true, EnableIPv6: true, BatchSize: 10, @@ -693,7 +693,7 @@ func Test_factoryDisposeWorker_releaseIP(t *testing.T) { func Test_commit_responsed(t *testing.T) { f := factorymocks.NewFactory(t) - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &daemon.PoolConfig{ EnableIPv4: true, EnableIPv6: true, BatchSize: 10, @@ -734,7 +734,7 @@ func Test_commit_responsed(t *testing.T) { func Test_commit_canceled(t *testing.T) { f := factorymocks.NewFactory(t) - local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &types.PoolConfig{ + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, f, &daemon.PoolConfig{ EnableIPv4: true, EnableIPv6: true, BatchSize: 10, diff --git a/pkg/eni/manager.go b/pkg/eni/manager.go index bbc81e8b..c9a3d120 100644 --- a/pkg/eni/manager.go +++ b/pkg/eni/manager.go @@ -96,7 +96,7 @@ func (n ByPriority) Swap(i, j int) { n[i], n[j] = n[j], n[i] } type Manager struct { sync.RWMutex networkInterfaces []NetworkInterface - selectionPolicy types.EniSelectionPolicy + selectionPolicy daemon.EniSelectionPolicy minIdles int maxIdles int @@ -161,7 +161,7 @@ func (m *Manager) Allocate(ctx context.Context, cni *daemon.CNI, req *AllocReque m.Lock() switch m.selectionPolicy { - case types.EniSelectionPolicyLeastIPs: + case daemon.EniSelectionPolicyLeastIPs: sort.Sort(sort.Reverse(ByPriority(m.networkInterfaces))) default: sort.Sort(ByPriority(m.networkInterfaces)) @@ -279,7 +279,7 @@ func (m *Manager) Status() []Status { func (m *Manager) syncPool(ctx context.Context) { m.Lock() switch m.selectionPolicy { - case types.EniSelectionPolicyLeastIPs: + case daemon.EniSelectionPolicyLeastIPs: sort.Sort(ByPriority(m.networkInterfaces)) default: sort.Sort(sort.Reverse(ByPriority(m.networkInterfaces))) @@ -355,7 +355,7 @@ func (m *Manager) syncPool(ctx context.Context) { wg.Wait() } -func NewManager(minIdles, maxIdles, total int, syncPeriod time.Duration, networkInterfaces []NetworkInterface, selectionPolicy types.EniSelectionPolicy, k8s k8s.Kubernetes) *Manager { +func NewManager(minIdles, maxIdles, total int, syncPeriod time.Duration, networkInterfaces []NetworkInterface, selectionPolicy daemon.EniSelectionPolicy, k8s k8s.Kubernetes) *Manager { if syncPeriod < 2*time.Minute && syncPeriod > 0 { syncPeriod = 2 * time.Minute } diff --git a/pkg/eni/manager_test.go b/pkg/eni/manager_test.go index b3cded83..19944bcd 100644 --- a/pkg/eni/manager_test.go +++ b/pkg/eni/manager_test.go @@ -84,7 +84,7 @@ func (s *success) Run(ctx context.Context, podResources []daemon.PodResources, w func TestManagerAllocateReturnsResourcesWhenSuccessful(t *testing.T) { mockNI := &success{} - manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI}, types.EniSelectionPolicyMostIPs, &FakeK8s{}) + manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI}, daemon.EniSelectionPolicyMostIPs, &FakeK8s{}) request := NewLocalIPRequest() resources, err := manager.Allocate(context.Background(), &daemon.CNI{}, &AllocRequest{ @@ -108,7 +108,7 @@ func TestManagerAllocateSelectionPolicy(t *testing.T) { } { - manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI, mockNI2}, types.EniSelectionPolicyMostIPs, &FakeK8s{}) + manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI, mockNI2}, daemon.EniSelectionPolicyMostIPs, &FakeK8s{}) request := NewLocalIPRequest() resources, err := manager.Allocate(context.Background(), &daemon.CNI{}, &AllocRequest{ @@ -121,7 +121,7 @@ func TestManagerAllocateSelectionPolicy(t *testing.T) { } { - manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI, mockNI2}, types.EniSelectionPolicyLeastIPs, &FakeK8s{}) + manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI, mockNI2}, daemon.EniSelectionPolicyLeastIPs, &FakeK8s{}) request := NewLocalIPRequest() resources, err := manager.Allocate(context.Background(), &daemon.CNI{}, &AllocRequest{ @@ -135,7 +135,7 @@ func TestManagerAllocateSelectionPolicy(t *testing.T) { } func TestManagerAllocateReturnsErrorWhenNoBackendCanHandleAllocation(t *testing.T) { - manager := NewManager(0, 0, 0, 0, []NetworkInterface{}, types.EniSelectionPolicyMostIPs, &FakeK8s{}) + manager := NewManager(0, 0, 0, 0, []NetworkInterface{}, daemon.EniSelectionPolicyMostIPs, &FakeK8s{}) request := NewLocalIPRequest() _, err := manager.Allocate(context.Background(), &daemon.CNI{}, &AllocRequest{ @@ -147,7 +147,7 @@ func TestManagerAllocateReturnsErrorWhenNoBackendCanHandleAllocation(t *testing. func TestManagerAllocateWithTimeoutWhenAllocationFails(t *testing.T) { mockNI := &timeOut{} - manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI}, types.EniSelectionPolicyMostIPs, &FakeK8s{}) + manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI}, daemon.EniSelectionPolicyMostIPs, &FakeK8s{}) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() diff --git a/pkg/eni/node_reconcile.go b/pkg/eni/node_reconcile.go index 0d3b2044..77b93e98 100644 --- a/pkg/eni/node_reconcile.go +++ b/pkg/eni/node_reconcile.go @@ -108,7 +108,7 @@ func (r *nodeReconcile) Reconcile(ctx context.Context, request reconcile.Request if len(vswitchOptions) == 0 { // if user forget to set vsw , we still rely on metadata to get the actual one - switchID, err := instance.VSwitchID() + switchID, err := instance.GetInstanceMeta().GetVSwitchID() if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get vsw from metadata, %w", err) } @@ -227,27 +227,27 @@ func (r *nodeReconcile) handleEFLO(ctx context.Context, k8sNode *corev1.Node, no if node.Labels == nil { node.Labels = map[string]string{} } - node.Labels[types.LinJunNodeLabel] = "true" + node.Labels[types.LinJunNodeLabelKey] = "true" - regionID, err := instance.EFLORegionID() + regionID, err := instance.GetInstanceMeta().GetRegionID() if err != nil { return reconcile.Result{}, err } - instanceType, err := instance.EFLOInstanceType() + instanceType, err := instance.GetInstanceMeta().GetInstanceType() if err != nil { return reconcile.Result{}, err } - nodeID, err := instance.EFLONodeID() + instanceID, err := instance.GetInstanceMeta().GetInstanceID() if err != nil { return reconcile.Result{}, err } - zoneID, err := instance.EFLOZoneID() + zoneID, err := instance.GetInstanceMeta().GetZoneID() if err != nil { return reconcile.Result{}, err } node.Spec.NodeMetadata.RegionID = regionID node.Spec.NodeMetadata.InstanceType = instanceType - node.Spec.NodeMetadata.InstanceID = nodeID + node.Spec.NodeMetadata.InstanceID = instanceID node.Spec.NodeMetadata.ZoneID = zoneID vswitchOptions := []string{} @@ -257,7 +257,7 @@ func (r *nodeReconcile) handleEFLO(ctx context.Context, k8sNode *corev1.Node, no } } if len(vswitchOptions) == 0 { - return reconcile.Result{}, fmt.Errorf("failed to get vsw for zone %s, %w", zoneID, err) + return reconcile.Result{}, fmt.Errorf("failed to get vsw for zone %s", zoneID) } policy := networkv1beta1.VSwitchSelectionPolicyRandom diff --git a/pkg/eni/node_reconcile_test.go b/pkg/eni/node_reconcile_test.go new file mode 100644 index 00000000..e057c980 --- /dev/null +++ b/pkg/eni/node_reconcile_test.go @@ -0,0 +1,107 @@ +package eni + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/AliyunContainerService/terway/pkg/aliyun/instance" + "github.com/AliyunContainerService/terway/pkg/aliyun/instance/mocks" + networkv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" +) + +var _ = Describe("Node controller", func() { + Context("Create Node", func() { + const ( + nodeName = "foo" + ) + + It("New EFLO node", func() { + ctx := context.Background() + ins := mocks.NewInterface(GinkgoT()) + ins.On("GetInstanceID").Return("i-foo", nil) + ins.On("GetRegionID").Return("cn-hangzhou", nil) + ins.On("GetInstanceType").Return("ecs", nil) + ins.On("GetZoneID").Return("cn-hangzhou-i", nil) + instance.Init(ins) + + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{ + "alibabacloud.com/lingjun-worker": "true", + }, + }, + Spec: corev1.NodeSpec{ + ProviderID: "", + }, + } + Expect(k8sClient.Create(ctx, k8sNode)).Should(Succeed()) + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "eni-config", + Namespace: "kube-system", + }, + Data: map[string]string{ + "eni_conf": "{ \"vswitches\": {\"cn-hangzhou-i\":[\"vsw-xx\"] } ,\"security_group\":\"sg-x\" , \"vswitch_selection_policy\": \"ordered\" }", + }} + Expect(k8sClient.Create(ctx, cm)).Should(Succeed()) + + By("Reconciling the created resource") + controllerReconciler := &nodeReconcile{ + client: k8sClient, + nodeName: nodeName, + record: record.NewFakeRecorder(100), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: nodeName}, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Add networkv1beta1.node") + node := &networkv1beta1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Spec: networkv1beta1.NodeSpec{ + NodeMetadata: networkv1beta1.NodeMetadata{ + RegionID: "cn-hangzhou", + InstanceID: "i-foo", + InstanceType: "ecs", + ZoneID: "cn-hangzhou-i", + }, + ENISpec: &networkv1beta1.ENISpec{ + VSwitchOptions: []string{"vsw-xx"}, + SecurityGroupIDs: []string{"foo"}, + VSwitchSelectPolicy: "ordered", + }, + }, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + + By("Reconciling the created resource") + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: nodeName}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, types.NamespacedName{Name: nodeName}, node) + Expect(err).NotTo(HaveOccurred()) + + Expect(node.Labels["alibabacloud.com/lingjun-worker"]).To(Equal("true")) + Expect(node.Spec.NodeMetadata.RegionID).To(Equal("cn-hangzhou")) + Expect(node.Spec.NodeMetadata.InstanceType).To(Equal("ecs")) + Expect(node.Spec.NodeMetadata.InstanceID).To(Equal("i-foo")) + Expect(node.Spec.NodeMetadata.ZoneID).To(Equal("cn-hangzhou-i")) + }) + }) +}) diff --git a/pkg/eni/suite_test.go b/pkg/eni/suite_test.go new file mode 100644 index 00000000..ff8efeb5 --- /dev/null +++ b/pkg/eni/suite_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eni + +import ( + "context" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + networkv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + testEnv *envtest.Environment + cfg *rest.Config + k8sClient client.Client +) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + err := networkv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "apis", "crds")}, + ErrorIfCRDPathMissing: true, + } + + // Retrieve the first found binary directory to allow running tests from IDEs + if getFirstFoundEnvTestBinaryDir() != "" { + testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +func getFirstFoundEnvTestBinaryDir() string { + basePath := filepath.Join("..", "..", "bin", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/pkg/factory/aliyun/aliyun.go b/pkg/factory/aliyun/aliyun.go index bff22c97..c0ddeb9a 100644 --- a/pkg/factory/aliyun/aliyun.go +++ b/pkg/factory/aliyun/aliyun.go @@ -18,7 +18,6 @@ import ( "github.com/AliyunContainerService/terway/pkg/backoff" "github.com/AliyunContainerService/terway/pkg/factory" vswpool "github.com/AliyunContainerService/terway/pkg/vswitch" - "github.com/AliyunContainerService/terway/types" "github.com/AliyunContainerService/terway/types/daemon" ) @@ -53,11 +52,11 @@ type Aliyun struct { eniTags map[string]string - eniTypeAttr types.Feat + eniTypeAttr daemon.Feat eniTagFilter map[string]string } -func NewAliyun(ctx context.Context, openAPI *client.OpenAPI, getter eni.ENIInfoGetter, vsw *vswpool.SwitchPool, cfg *types.ENIConfig) *Aliyun { +func NewAliyun(ctx context.Context, openAPI *client.OpenAPI, getter eni.ENIInfoGetter, vsw *vswpool.SwitchPool, cfg *daemon.ENIConfig) *Aliyun { return &Aliyun{ ctx: ctx, @@ -456,7 +455,7 @@ func (a *Aliyun) GetAttachedNetworkInterface(trunkENIID string) ([]*daemon.ENI, if trunkENIID == eni.ID { eni.Trunk = true - types.DisableFeature(&feat, types.FeatTrunk) + daemon.DisableFeature(&feat, daemon.FeatTrunk) } } diff --git a/pkg/factory/aliyun/eflo.go b/pkg/factory/aliyun/eflo.go index 88ba7f52..e1ba29f4 100644 --- a/pkg/factory/aliyun/eflo.go +++ b/pkg/factory/aliyun/eflo.go @@ -37,7 +37,7 @@ type Eflo struct { selectionPolicy vswpool.SelectionPolicy } -func NewEflo(ctx context.Context, openAPI *client.OpenAPI, vsw *vswpool.SwitchPool, cfg *types.ENIConfig) *Eflo { +func NewEflo(ctx context.Context, openAPI *client.OpenAPI, vsw *vswpool.SwitchPool, cfg *daemon.ENIConfig) *Eflo { return &Eflo{ ctx: ctx, api: openAPI, diff --git a/pkg/factory/mocks/Factory.go b/pkg/factory/mocks/Factory.go index 4a0f8ab5..bfbc8760 100644 --- a/pkg/factory/mocks/Factory.go +++ b/pkg/factory/mocks/Factory.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.52.2. DO NOT EDIT. package mocks diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 3ea1c723..be942ff9 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -14,8 +14,11 @@ func init() { const ( // AutoDataPathV2 enable the new datapath feature. AutoDataPathV2 featuregate.Feature = "AutoDataPathV2" + + EFLO featuregate.Feature = "EFLO" ) var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ AutoDataPathV2: {Default: true, PreRelease: featuregate.Alpha}, + EFLO: {Default: false, PreRelease: featuregate.Alpha}, } diff --git a/pkg/k8s/mocks/Kubernetes.go b/pkg/k8s/mocks/Kubernetes.go index 12fa3204..1efb1351 100644 --- a/pkg/k8s/mocks/Kubernetes.go +++ b/pkg/k8s/mocks/Kubernetes.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.52.2. DO NOT EDIT. package mocks @@ -21,7 +21,7 @@ type Kubernetes struct { mock.Mock } -// GetClient provides a mock function with given fields: +// GetClient provides a mock function with no fields func (_m *Kubernetes) GetClient() client.Client { ret := _m.Called() @@ -69,7 +69,7 @@ func (_m *Kubernetes) GetDynamicConfigWithName(ctx context.Context, name string) return r0, r1 } -// GetLocalPods provides a mock function with given fields: +// GetLocalPods provides a mock function with no fields func (_m *Kubernetes) GetLocalPods() ([]*daemon.PodInfo, error) { ret := _m.Called() @@ -99,7 +99,7 @@ func (_m *Kubernetes) GetLocalPods() ([]*daemon.PodInfo, error) { return r0, r1 } -// GetNodeDynamicConfigLabel provides a mock function with given fields: +// GetNodeDynamicConfigLabel provides a mock function with no fields func (_m *Kubernetes) GetNodeDynamicConfigLabel() string { ret := _m.Called() @@ -147,7 +147,7 @@ func (_m *Kubernetes) GetPod(ctx context.Context, namespace string, name string, return r0, r1 } -// GetServiceCIDR provides a mock function with given fields: +// GetServiceCIDR provides a mock function with no fields func (_m *Kubernetes) GetServiceCIDR() *types.IPNetSet { ret := _m.Called() @@ -167,7 +167,7 @@ func (_m *Kubernetes) GetServiceCIDR() *types.IPNetSet { return r0 } -// GetTrunkID provides a mock function with given fields: +// GetTrunkID provides a mock function with no fields func (_m *Kubernetes) GetTrunkID() string { ret := _m.Called() @@ -185,7 +185,7 @@ func (_m *Kubernetes) GetTrunkID() string { return r0 } -// Node provides a mock function with given fields: +// Node provides a mock function with no fields func (_m *Kubernetes) Node() *v1.Node { ret := _m.Called() @@ -205,7 +205,7 @@ func (_m *Kubernetes) Node() *v1.Node { return r0 } -// NodeName provides a mock function with given fields: +// NodeName provides a mock function with no fields func (_m *Kubernetes) NodeName() string { ret := _m.Called() diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index e1d1090c..2b953a7d 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -8,6 +8,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" + "github.com/AliyunContainerService/terway/types" ) var stsKinds = []string{"StatefulSet"} @@ -48,7 +49,7 @@ func ISVKNode(n *corev1.Node) bool { } func ISLinJunNode(lb map[string]string) bool { - return lb["alibabacloud.com/lingjun-worker"] == "true" + return lb[types.LinJunNodeLabelKey] == "true" } // PodSandboxExited pod sandbox is exited diff --git a/pkg/utils/nodecap/mocks/NodeCapabilitiesStore.go b/pkg/utils/nodecap/mocks/NodeCapabilitiesStore.go index 91297d36..84b1de40 100644 --- a/pkg/utils/nodecap/mocks/NodeCapabilitiesStore.go +++ b/pkg/utils/nodecap/mocks/NodeCapabilitiesStore.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.45.0. DO NOT EDIT. +// Code generated by mockery v2.52.2. DO NOT EDIT. package mocks @@ -27,7 +27,7 @@ func (_m *NodeCapabilitiesStore) Get(capName string) string { return r0 } -// Load provides a mock function with given fields: +// Load provides a mock function with no fields func (_m *NodeCapabilitiesStore) Load() error { ret := _m.Called() @@ -45,7 +45,7 @@ func (_m *NodeCapabilitiesStore) Load() error { return r0 } -// Save provides a mock function with given fields: +// Save provides a mock function with no fields func (_m *NodeCapabilitiesStore) Save() error { ret := _m.Called() diff --git a/types/config.go b/types/config.go deleted file mode 100644 index 63f228c5..00000000 --- a/types/config.go +++ /dev/null @@ -1,68 +0,0 @@ -package types - -import ( - "github.com/AliyunContainerService/terway/pkg/vswitch" -) - -type EniSelectionPolicy string - -// Network interface Selection Policy -const ( - EniSelectionPolicyLeastIPs EniSelectionPolicy = "least_ips" - EniSelectionPolicyMostIPs EniSelectionPolicy = "most_ips" -) - -type ENIConfig struct { - ZoneID string - VSwitchOptions []string - ENITags map[string]string - SecurityGroupIDs []string - InstanceID string - - VSwitchSelectionPolicy vswitch.SelectionPolicy - EniSelectionPolicy EniSelectionPolicy - - ResourceGroupID string - - EniTypeAttr Feat - - EnableIPv4 bool - EnableIPv6 bool - - TagFilter map[string]string -} - -// PoolConfig configuration of pool and resource factory -type PoolConfig struct { - EnableIPv4 bool - EnableIPv6 bool - - Capacity int // the max res can hold in the pool - MaxENI int // the max eni terway can be created (already exclude main eni) - MaxMemberENI int // the max member eni can be created - ERdmaCapacity int // the max erdma res can be created - MaxIPPerENI int - BatchSize int - - MaxPoolSize int - MinPoolSize int -} - -type Feat uint8 - -const ( - FeatTrunk Feat = 1 << iota - FeatERDMA -) - -func EnableFeature(features *Feat, feature Feat) { - *features |= feature -} - -func DisableFeature(features *Feat, feature Feat) { - *features &= ^feature -} - -func IsFeatureEnabled(features Feat, feature Feat) bool { - return features&feature != 0 -} diff --git a/types/daemon/config.go b/types/daemon/config.go index cfcc4e87..b4b6bb03 100644 --- a/types/daemon/config.go +++ b/types/daemon/config.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" + "github.com/AliyunContainerService/terway/pkg/vswitch" "github.com/AliyunContainerService/terway/types/secret" jsonpatch "github.com/evanphx/json-patch" @@ -180,3 +181,66 @@ func GetAddonSecret() (string, string, error) { } return string(keyID), string(keySecret), nil } + +type EniSelectionPolicy string + +// Network interface Selection Policy +const ( + EniSelectionPolicyLeastIPs EniSelectionPolicy = "least_ips" + EniSelectionPolicyMostIPs EniSelectionPolicy = "most_ips" +) + +type ENIConfig struct { + ZoneID string + VSwitchOptions []string + ENITags map[string]string + SecurityGroupIDs []string + InstanceID string + + VSwitchSelectionPolicy vswitch.SelectionPolicy + EniSelectionPolicy EniSelectionPolicy + + ResourceGroupID string + + EniTypeAttr Feat + + EnableIPv4 bool + EnableIPv6 bool + + TagFilter map[string]string +} + +// PoolConfig configuration of pool and resource factory +type PoolConfig struct { + EnableIPv4 bool + EnableIPv6 bool + + Capacity int // the max res can hold in the pool + MaxENI int // the max eni terway can be created (already exclude main eni) + MaxMemberENI int // the max member eni can be created + ERdmaCapacity int // the max erdma res can be created + MaxIPPerENI int + BatchSize int + + MaxPoolSize int + MinPoolSize int +} + +type Feat uint8 + +const ( + FeatTrunk Feat = 1 << iota + FeatERDMA +) + +func EnableFeature(features *Feat, feature Feat) { + *features |= feature +} + +func DisableFeature(features *Feat, feature Feat) { + *features &= ^feature +} + +func IsFeatureEnabled(features Feat, feature Feat) bool { + return features&feature != 0 +} diff --git a/types/k8s.go b/types/k8s.go index a83d2d8c..033f81af 100644 --- a/types/k8s.go +++ b/types/k8s.go @@ -68,6 +68,8 @@ const ( IgnoreByTerway = LabelPrefix + "ignore-by-terway" ExclusiveENIModeLabel = LabelPrefix + "exclusive-mode-eni-type" + + LinJunNodeLabelKey = "alibabacloud.com/lingjun-worker" ) // FinalizerPodENI finalizer for podENI resource