Refactor(instance): Refactor the way instance metadata is retrieved

- Modify the logic for retrieving instance metadata, optimizing with interfaces and caching
- Refactor the way EFLO instance metadata is retrieved
- Update relevant unit tests

Signed-off-by: l1b0k <[email protected]>

cmd/terway-cli/cni_linux.go

@@ -3,8 +3,10 @@ package main
 import (
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
+	"syscall"
 
-	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/vishvananda/netlink"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 
@@ -60,6 +62,54 @@ func allowEBPFNetworkPolicy(require bool) (bool, error) {
 	return require, nil
 }
 
-func checkKernelVersion(k, major, minor int) bool {
-	return kernel.CheckKernelVersion(k, major, minor)
+func checkKernelVersion(iMajor, iMinor, iPatch int) bool {
+	var un syscall.Utsname
+	syscall.Uname(&un)
+	var sb strings.Builder
+	for _, b := range un.Release[:] {
+		if b == 0 {
+			break
+		}
+		sb.WriteByte(byte(b))
+	}
+	major, minor, patch, ok := parseRelease(sb.String())
+	return ok && (major > iMajor ||
+		major == iMajor && minor > iMinor ||
+		major == iMajor && minor == iMinor && iPatch >= patch)
+}
+
+// parseRelease parses a dot-separated version number. It follows the semver
+// syntax, but allows the minor and patch versions to be elided.
+//
+// This is a copy of the Go runtime's parseRelease from
+// https://golang.org/cl/209597.
+func parseRelease(rel string) (major, minor, patch int, ok bool) {
+	// Strip anything after a dash or plus.
+	for i := 0; i < len(rel); i++ {
+		if rel[i] == '-' || rel[i] == '+' {
+			rel = rel[:i]
+			break
+		}
+	}
+
+	next := func() (int, bool) {
+		for i := 0; i < len(rel); i++ {
+			if rel[i] == '.' {
+				ver, err := strconv.Atoi(rel[:i])
+				rel = rel[i+1:]
+				return ver, err == nil
+			}
+		}
+		ver, err := strconv.Atoi(rel)
+		rel = ""
+		return ver, err == nil
+	}
+	if major, ok = next(); !ok || rel == "" {
+		return
+	}
+	if minor, ok = next(); !ok || rel == "" {
+		return
+	}
+	patch, ok = next()
+	return
 }

daemon/builder.go

@@ -39,7 +39,8 @@ type NetworkServiceBuilder struct {
 
 	limit *client.Limits
 
-	err error
+	eflo bool
+	err  error
 }
 
 func NewNetworkServiceBuilder(ctx context.Context) *NetworkServiceBuilder {
@@ -120,6 +121,10 @@ func (b *NetworkServiceBuilder) InitK8S() *NetworkServiceBuilder {
 		b.service.daemonMode = daemon.ModeENIOnly
 	}
 
+	if utils.ISLinJunNode(b.service.k8s.Node().Labels) {
+		b.eflo = true
+		instance.Init(&instance.EFLO{})
+	}
 	return b
 }
 
@@ -152,10 +157,10 @@ func (b *NetworkServiceBuilder) LoadDynamicConfig() *NetworkServiceBuilder {
 }
 
 func (b *NetworkServiceBuilder) setupAliyunClient() error {
-	if os.Getenv("TERWAY_DEPLOY_ENV") == envEFLO {
-		instance.SetPopulateFunc(instance.EfloPopulate)
+	regionID, err := instance.GetInstanceMeta().GetRegionID()
+	if err != nil {
+		return err
 	}
-	meta := instance.GetInstanceMeta()
 
 	var providers []credential.Interface
 	if string(b.config.AccessID) != "" && string(b.config.AccessSecret) != "" {
@@ -164,7 +169,7 @@ func (b *NetworkServiceBuilder) setupAliyunClient() error {
 	providers = append(providers, credential.NewEncryptedCredentialProvider(utils.NormalizePath(b.config.CredentialPath)))
 	providers = append(providers, credential.NewMetadataProvider())
 
-	clientSet, err := credential.NewClientMgr(meta.RegionID, providers...)
+	clientSet, err := credential.NewClientMgr(regionID, providers...)
 	if err != nil {
 		return err
 	}
@@ -184,14 +189,18 @@ func (b *NetworkServiceBuilder) initInstanceLimit() error {
 		return fmt.Errorf("k8s node not found")
 	}
 	provider := client.LimitProviders["ecs"]
-	if os.Getenv("TERWAY_DEPLOY_ENV") == envEFLO {
+	if b.eflo {
 		provider = client.LimitProviders["eflo"]
 		limit, err := provider.GetLimitFromAnno(node.Annotations)
 		if err != nil {
 			return err
 		}
 		if limit == nil {
-			limit, err = provider.GetLimit(b.aliyunClient, instance.GetInstanceMeta().InstanceID)
+			instanceID, err := instance.GetInstanceMeta().GetInstanceID()
+			if err != nil {
+				return err
+			}
+			limit, err = provider.GetLimit(b.aliyunClient, instanceID)
 			if err != nil {
 				return fmt.Errorf("upable get instance limit, %w", err)
 			}
@@ -202,10 +211,28 @@ func (b *NetworkServiceBuilder) initInstanceLimit() error {
 		if err != nil {
 			return err
 		}
-		if limit == nil || instance.GetInstanceMeta().InstanceType != limit.InstanceTypeID {
-			limit, err = provider.GetLimit(b.aliyunClient, instance.GetInstanceMeta().InstanceType)
+
+		if limit != nil {
+			instanceType, err := instance.GetInstanceMeta().GetInstanceType()
 			if err != nil {
-				return fmt.Errorf("upable get instance limit, %w", err)
+				return err
+			}
+			if limit.InstanceTypeID != instanceType {
+				limit = nil
+			}
+		}
+
+		if limit == nil {
+			instanceType, err := instance.GetInstanceMeta().GetInstanceType()
+			if err != nil {
+				return err
+			}
+
+			if instanceType != limit.InstanceTypeID {
+				limit, err = provider.GetLimit(b.aliyunClient, instanceType)
+				if err != nil {
+					return fmt.Errorf("upable get instance limit, %w", err)
+				}
 			}
 		}
 		b.limit = limit
@@ -227,6 +254,29 @@ func (b *NetworkServiceBuilder) setupENIManager() error {
 	eniConfig.EnableIPv4 = enableIPv4
 	eniConfig.EnableIPv6 = enableIPv6
 
+	zoneID, err := instance.GetInstanceMeta().GetZoneID()
+	if err != nil {
+		return err
+	}
+	instanceID, err := instance.GetInstanceMeta().GetInstanceID()
+	if err != nil {
+		return err
+	}
+	vswitchID, err := instance.GetInstanceMeta().GetVSwitchID()
+	if err != nil {
+		return err
+	}
+
+	if eniConfig.ZoneID == "" {
+		eniConfig.ZoneID = zoneID
+	}
+	if eniConfig.InstanceID == "" {
+		eniConfig.InstanceID = instanceID
+	}
+	if len(eniConfig.VSwitchOptions) == 0 {
+		eniConfig.VSwitchOptions = []string{vswitchID}
+	}
+
 	// fall back to use primary eni's sg
 	if len(eniConfig.SecurityGroupIDs) == 0 {
 		enis, err := b.aliyunClient.DescribeNetworkInterface(b.ctx, "", nil, eniConfig.InstanceID, "Primary", "", nil)
@@ -361,7 +411,7 @@ func (b *NetworkServiceBuilder) setupENIManager() error {
 		eniList = append(eniList, eni.NewLocal(nil, "secondary", factory, poolConfig))
 	}
 
-	eniManager := eni.NewManager(poolConfig.MinPoolSize, poolConfig.MaxPoolSize, poolConfig.Capacity, 30*time.Second, eniList, types.EniSelectionPolicy(b.config.EniSelectionPolicy), b.service.k8s)
+	eniManager := eni.NewManager(poolConfig.MinPoolSize, poolConfig.MaxPoolSize, poolConfig.Capacity, 30*time.Second, eniList, daemon.EniSelectionPolicy(b.config.EniSelectionPolicy), b.service.k8s)
 	b.service.eniMgr = eniManager
 	err = eniManager.Run(b.ctx, &b.service.wg, podResources)
 	if err != nil {
@@ -405,7 +455,7 @@ func (b *NetworkServiceBuilder) PostInitForCRDV2() *NetworkServiceBuilder {
 		return b
 	}
 	crdv2 := eni.NewCRDV2(b.service.k8s.NodeName(), b.namespace)
-	mgr := eni.NewManager(0, 0, 0, 0, []eni.NetworkInterface{crdv2}, types.EniSelectionPolicy(b.config.EniSelectionPolicy), nil)
+	mgr := eni.NewManager(0, 0, 0, 0, []eni.NetworkInterface{crdv2}, daemon.EniSelectionPolicy(b.config.EniSelectionPolicy), nil)
 
 	svc := b.RunENIMgr(b.ctx, mgr)
 	go b.service.startGarbageCollectionLoop(b.ctx)

daemon/config.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"github.com/AliyunContainerService/terway/pkg/aliyun/client"
-	"github.com/AliyunContainerService/terway/pkg/aliyun/instance"
 	"github.com/AliyunContainerService/terway/pkg/k8s"
 	"github.com/AliyunContainerService/terway/pkg/utils"
 	"github.com/AliyunContainerService/terway/pkg/vswitch"
@@ -25,26 +24,24 @@ func getDynamicConfig(ctx context.Context, k8s k8s.Kubernetes) (string, string,
 	return cfg, label, err
 }
 
-func getENIConfig(cfg *daemon.Config) *types.ENIConfig {
+func getENIConfig(cfg *daemon.Config) *daemon.ENIConfig {
 	vswitchSelectionPolicy := vswitch.VSwitchSelectionPolicyRandom
 	switch cfg.VSwitchSelectionPolicy {
 	case "ordered":
 		// keep the previous behave
 		vswitchSelectionPolicy = vswitch.VSwitchSelectionPolicyMost
 	}
 
-	eniSelectionPolicy := types.EniSelectionPolicyMostIPs
+	eniSelectionPolicy := daemon.EniSelectionPolicyMostIPs
 	switch cfg.EniSelectionPolicy {
 	case "least_ips":
-		eniSelectionPolicy = types.EniSelectionPolicyLeastIPs
+		eniSelectionPolicy = daemon.EniSelectionPolicyLeastIPs
 	}
 
-	eniConfig := &types.ENIConfig{
-		ZoneID:                 instance.GetInstanceMeta().ZoneID,
+	eniConfig := &daemon.ENIConfig{
 		VSwitchOptions:         nil,
 		ENITags:                cfg.ENITags,
 		SecurityGroupIDs:       cfg.GetSecurityGroups(),
-		InstanceID:             instance.GetInstanceMeta().InstanceID,
 		VSwitchSelectionPolicy: vswitchSelectionPolicy,
 		EniSelectionPolicy:     eniSelectionPolicy,
 		ResourceGroupID:        cfg.ResourceGroupID,
@@ -59,24 +56,20 @@ func getENIConfig(cfg *daemon.Config) *types.ENIConfig {
 		}
 	}
 
-	if len(eniConfig.VSwitchOptions) == 0 {
-		eniConfig.VSwitchOptions = []string{instance.GetInstanceMeta().VSwitchID}
-	}
-
 	if cfg.EnableENITrunking {
-		types.EnableFeature(&eniConfig.EniTypeAttr, types.FeatTrunk)
+		daemon.EnableFeature(&eniConfig.EniTypeAttr, daemon.FeatTrunk)
 	}
 	if cfg.EnableERDMA {
-		types.EnableFeature(&eniConfig.EniTypeAttr, types.FeatERDMA)
+		daemon.EnableFeature(&eniConfig.EniTypeAttr, daemon.FeatERDMA)
 	}
 
 	return eniConfig
 }
 
 // the actual size for pool is minIdle and maxIdle
-func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *client.Limits) (*types.PoolConfig, error) {
+func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *client.Limits) (*daemon.PoolConfig, error) {
 
-	poolConfig := &types.PoolConfig{
+	poolConfig := &daemon.PoolConfig{
 		BatchSize: 10,
 	}
 

daemon/config_test.go

@@ -8,25 +8,51 @@ import (
 	"github.com/AliyunContainerService/terway/pkg/aliyun/client"
 	"github.com/AliyunContainerService/terway/pkg/aliyun/instance"
 	"github.com/AliyunContainerService/terway/pkg/vswitch"
-	"github.com/AliyunContainerService/terway/types"
 	"github.com/AliyunContainerService/terway/types/daemon"
 )
 
-func init() {
-	instance.SetPopulateFunc(func() *instance.Instance {
-		return &instance.Instance{
-			RegionID:     "regionID",
-			ZoneID:       "zoneID",
-			VPCID:        "vpc",
-			VSwitchID:    "vsw",
-			PrimaryMAC:   "",
-			InstanceID:   "instanceID",
-			InstanceType: "",
-		}
-	})
+type Mock struct {
+	regionID     string
+	zoneID       string
+	vSwitchID    string
+	primaryMAC   string
+	instanceID   string
+	instanceType string
+}
+
+func (m *Mock) GetRegionID() (string, error) {
+	return m.regionID, nil
+}
+
+func (m *Mock) GetZoneID() (string, error) {
+	return m.zoneID, nil
+}
+
+func (m *Mock) GetVSwitchID() (string, error) {
+	return m.vSwitchID, nil
+}
+
+func (m *Mock) GetPrimaryMAC() (string, error) {
+	return m.primaryMAC, nil
+}
+
+func (m *Mock) GetInstanceID() (string, error) {
+	return m.instanceID, nil
+}
+
+func (m *Mock) GetInstanceType() (string, error) {
+	return m.instanceType, nil
 }
 
 func TestGetPoolConfigWithENIMultiIPMode(t *testing.T) {
+	instance.Init(&Mock{
+		regionID:     "regionID",
+		zoneID:       "zoneID",
+		vSwitchID:    "vsw",
+		primaryMAC:   "",
+		instanceID:   "instanceID",
+		instanceType: "",
+	})
 	cfg := &daemon.Config{
 		MaxPoolSize: 5,
 		MinPoolSize: 1,
@@ -46,6 +72,14 @@ func TestGetPoolConfigWithENIMultiIPMode(t *testing.T) {
 }
 
 func TestGetENIConfig(t *testing.T) {
+	instance.Init(&Mock{
+		regionID:     "regionID",
+		zoneID:       "zoneID",
+		vSwitchID:    "vsw",
+		primaryMAC:   "",
+		instanceID:   "instanceID",
+		instanceType: "",
+	})
 	cfg := &daemon.Config{
 		ENITags:                map[string]string{"aa": "bb"},
 		SecurityGroups:         []string{"sg1", "sg2"},
@@ -61,13 +95,10 @@ func TestGetENIConfig(t *testing.T) {
 
 	eniConfig := getENIConfig(cfg)
 
-	assert.Equal(t, "zoneID", eniConfig.ZoneID)
-	assert.Equal(t, []string{"vswitch1", "vswitch2"}, eniConfig.VSwitchOptions)
 	assert.Equal(t, 1, len(eniConfig.ENITags))
 	assert.Equal(t, []string{"sg1", "sg2"}, eniConfig.SecurityGroupIDs)
-	assert.Equal(t, "instanceID", eniConfig.InstanceID)
 	assert.Equal(t, vswitch.VSwitchSelectionPolicyMost, eniConfig.VSwitchSelectionPolicy)
-	assert.Equal(t, types.EniSelectionPolicyMostIPs, eniConfig.EniSelectionPolicy)
+	assert.Equal(t, daemon.EniSelectionPolicyMostIPs, eniConfig.EniSelectionPolicy)
 	assert.Equal(t, "rgID", eniConfig.ResourceGroupID)
-	assert.Equal(t, types.Feat(3), eniConfig.EniTypeAttr)
+	assert.Equal(t, daemon.Feat(3), eniConfig.EniTypeAttr)
 }

daemon/daemon.go

@@ -850,7 +850,7 @@ func checkInstance(limit *client.Limits, daemonMode string, config *daemon.Confi
 }
 
 // initTrunk to ensure trunk eni is present. Return eni id if found.
-func initTrunk(config *daemon.Config, poolConfig *types.PoolConfig, k8sClient k8s.Kubernetes, f factory.Factory) (string, error) {
+func initTrunk(config *daemon.Config, poolConfig *daemon.PoolConfig, k8sClient k8s.Kubernetes, f factory.Factory) (string, error) {
 	var err error
 
 	// get eni id form node annotation
@@ -909,7 +909,7 @@ func initTrunk(config *daemon.Config, poolConfig *types.PoolConfig, k8sClient k8
 	return trunk.ID, nil
 }
 
-func runDevicePlugin(daemonMode string, config *daemon.Config, poolConfig *types.PoolConfig) {
+func runDevicePlugin(daemonMode string, config *daemon.Config, poolConfig *daemon.PoolConfig) {
 	switch daemonMode {
 	case daemon.ModeENIMultiIP:
 		if config.EnableENITrunking {