Refactor(instance): Refactor the way instance metadata is retrieved

- Modify the logic for retrieving instance metadata, optimizing with interfaces and caching
- Refactor the way EFLO instance metadata is retrieved
- Update relevant unit tests

Signed-off-by: l1b0k <[email protected]>

cmd/terway-cli/cni_linux.go

@@ -3,8 +3,10 @@ package main
 import (
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
+	"syscall"
 
-	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/vishvananda/netlink"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 
@@ -60,6 +62,54 @@ func allowEBPFNetworkPolicy(require bool) (bool, error) {
 	return require, nil
 }
 
-func checkKernelVersion(k, major, minor int) bool {
-	return kernel.CheckKernelVersion(k, major, minor)
+func checkKernelVersion(iMajor, iMinor, iPatch int) bool {
+	var un syscall.Utsname
+	syscall.Uname(&un)
+	var sb strings.Builder
+	for _, b := range un.Release[:] {
+		if b == 0 {
+			break
+		}
+		sb.WriteByte(byte(b))
+	}
+	major, minor, patch, ok := parseRelease(sb.String())
+	return ok && (major > iMajor ||
+		major == iMajor && minor > iMinor ||
+		major == iMajor && minor == iMinor && iPatch >= patch)
+}
+
+// parseRelease parses a dot-separated version number. It follows the semver
+// syntax, but allows the minor and patch versions to be elided.
+//
+// This is a copy of the Go runtime's parseRelease from
+// https://golang.org/cl/209597.
+func parseRelease(rel string) (major, minor, patch int, ok bool) {
+	// Strip anything after a dash or plus.
+	for i := 0; i < len(rel); i++ {
+		if rel[i] == '-' || rel[i] == '+' {
+			rel = rel[:i]
+			break
+		}
+	}
+
+	next := func() (int, bool) {
+		for i := 0; i < len(rel); i++ {
+			if rel[i] == '.' {
+				ver, err := strconv.Atoi(rel[:i])
+				rel = rel[i+1:]
+				return ver, err == nil
+			}
+		}
+		ver, err := strconv.Atoi(rel)
+		rel = ""
+		return ver, err == nil
+	}
+	if major, ok = next(); !ok || rel == "" {
+		return
+	}
+	if minor, ok = next(); !ok || rel == "" {
+		return
+	}
+	patch, ok = next()
+	return
 }

cmd/terway-controlplane/terway-controlplane.go

@@ -23,6 +23,7 @@ import (
 	"math/rand"
 	"net"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/samber/lo"
@@ -41,7 +42,9 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog/v2/textlogger"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -92,9 +95,12 @@ func main() {
 	var (
 		configFilePath     string
 		credentialFilePath string
+		featureGates       map[string]bool
 	)
 	flag.StringVar(&configFilePath, "config", "/etc/config/ctrl-config.yaml", "config file for controlplane")
 	flag.StringVar(&credentialFilePath, "credential", "/etc/credential/ctrl-secret.yaml", "secret file for controlplane")
+	flag.Var(cliflag.NewMapStringBool(&featureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
+		"Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n"))
 
 	logCfg := textlogger.NewConfig()
 	logCfg.AddFlags(flag.CommandLine)
@@ -104,6 +110,12 @@ func main() {
 	ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig()))
 	log.Info(version.Version)
 
+	err := utilfeature.DefaultMutableFeatureGate.SetFromMap(featureGates)
+	if err != nil {
+		log.Error(err, "unable to set feature gates")
+		os.Exit(1)
+	}
+
 	ctx := ctrl.SetupSignalHandler()
 
 	cfg, err := controlplane.ParseAndValidate(configFilePath, credentialFilePath)

daemon/builder.go

@@ -39,7 +39,8 @@ type NetworkServiceBuilder struct {
 
 	limit *client.Limits
 
-	err error
+	eflo bool
+	err  error
 }
 
 func NewNetworkServiceBuilder(ctx context.Context) *NetworkServiceBuilder {
@@ -120,6 +121,10 @@ func (b *NetworkServiceBuilder) InitK8S() *NetworkServiceBuilder {
 		b.service.daemonMode = daemon.ModeENIOnly
 	}
 
+	if utils.ISLinJunNode(b.service.k8s.Node().Labels) {
+		b.eflo = true
+		instance.Init(&instance.EFLO{})
+	}
 	return b
 }
 
@@ -152,10 +157,10 @@ func (b *NetworkServiceBuilder) LoadDynamicConfig() *NetworkServiceBuilder {
 }
 
 func (b *NetworkServiceBuilder) setupAliyunClient() error {
-	if os.Getenv("TERWAY_DEPLOY_ENV") == envEFLO {
-		instance.SetPopulateFunc(instance.EfloPopulate)
+	regionID, err := instance.GetInstanceMeta().GetRegionID()
+	if err != nil {
+		return err
 	}
-	meta := instance.GetInstanceMeta()
 
 	var providers []credential.Interface
 	if string(b.config.AccessID) != "" && string(b.config.AccessSecret) != "" {
@@ -164,7 +169,7 @@ func (b *NetworkServiceBuilder) setupAliyunClient() error {
 	providers = append(providers, credential.NewEncryptedCredentialProvider(utils.NormalizePath(b.config.CredentialPath)))
 	providers = append(providers, credential.NewMetadataProvider())
 
-	clientSet, err := credential.NewClientMgr(meta.RegionID, providers...)
+	clientSet, err := credential.NewClientMgr(regionID, providers...)
 	if err != nil {
 		return err
 	}
@@ -184,14 +189,18 @@ func (b *NetworkServiceBuilder) initInstanceLimit() error {
 		return fmt.Errorf("k8s node not found")
 	}
 	provider := client.LimitProviders["ecs"]
-	if os.Getenv("TERWAY_DEPLOY_ENV") == envEFLO {
+	if b.eflo {
 		provider = client.LimitProviders["eflo"]
 		limit, err := provider.GetLimitFromAnno(node.Annotations)
 		if err != nil {
 			return err
 		}
 		if limit == nil {
-			limit, err = provider.GetLimit(b.aliyunClient, instance.GetInstanceMeta().InstanceID)
+			instanceID, err := instance.GetInstanceMeta().GetInstanceID()
+			if err != nil {
+				return err
+			}
+			limit, err = provider.GetLimit(b.aliyunClient, instanceID)
 			if err != nil {
 				return fmt.Errorf("upable get instance limit, %w", err)
 			}
@@ -202,10 +211,28 @@ func (b *NetworkServiceBuilder) initInstanceLimit() error {
 		if err != nil {
 			return err
 		}
-		if limit == nil || instance.GetInstanceMeta().InstanceType != limit.InstanceTypeID {
-			limit, err = provider.GetLimit(b.aliyunClient, instance.GetInstanceMeta().InstanceType)
+
+		if limit != nil {
+			instanceType, err := instance.GetInstanceMeta().GetInstanceType()
 			if err != nil {
-				return fmt.Errorf("upable get instance limit, %w", err)
+				return err
+			}
+			if limit.InstanceTypeID != instanceType {
+				limit = nil
+			}
+		}
+
+		if limit == nil {
+			instanceType, err := instance.GetInstanceMeta().GetInstanceType()
+			if err != nil {
+				return err
+			}
+
+			if instanceType != limit.InstanceTypeID {
+				limit, err = provider.GetLimit(b.aliyunClient, instanceType)
+				if err != nil {
+					return fmt.Errorf("upable get instance limit, %w", err)
+				}
 			}
 		}
 		b.limit = limit
@@ -227,6 +254,29 @@ func (b *NetworkServiceBuilder) setupENIManager() error {
 	eniConfig.EnableIPv4 = enableIPv4
 	eniConfig.EnableIPv6 = enableIPv6
 
+	zoneID, err := instance.GetInstanceMeta().GetZoneID()
+	if err != nil {
+		return err
+	}
+	instanceID, err := instance.GetInstanceMeta().GetInstanceID()
+	if err != nil {
+		return err
+	}
+	vswitchID, err := instance.GetInstanceMeta().GetVSwitchID()
+	if err != nil {
+		return err
+	}
+
+	if eniConfig.ZoneID == "" {
+		eniConfig.ZoneID = zoneID
+	}
+	if eniConfig.InstanceID == "" {
+		eniConfig.InstanceID = instanceID
+	}
+	if len(eniConfig.VSwitchOptions) == 0 {
+		eniConfig.VSwitchOptions = []string{vswitchID}
+	}
+
 	// fall back to use primary eni's sg
 	if len(eniConfig.SecurityGroupIDs) == 0 {
 		enis, err := b.aliyunClient.DescribeNetworkInterface(b.ctx, "", nil, eniConfig.InstanceID, "Primary", "", nil)
@@ -361,7 +411,7 @@ func (b *NetworkServiceBuilder) setupENIManager() error {
 		eniList = append(eniList, eni.NewLocal(nil, "secondary", factory, poolConfig))
 	}
 
-	eniManager := eni.NewManager(poolConfig.MinPoolSize, poolConfig.MaxPoolSize, poolConfig.Capacity, 30*time.Second, eniList, types.EniSelectionPolicy(b.config.EniSelectionPolicy), b.service.k8s)
+	eniManager := eni.NewManager(poolConfig.MinPoolSize, poolConfig.MaxPoolSize, poolConfig.Capacity, 30*time.Second, eniList, daemon.EniSelectionPolicy(b.config.EniSelectionPolicy), b.service.k8s)
 	b.service.eniMgr = eniManager
 	err = eniManager.Run(b.ctx, &b.service.wg, podResources)
 	if err != nil {
@@ -405,7 +455,7 @@ func (b *NetworkServiceBuilder) PostInitForCRDV2() *NetworkServiceBuilder {
 		return b
 	}
 	crdv2 := eni.NewCRDV2(b.service.k8s.NodeName(), b.namespace)
-	mgr := eni.NewManager(0, 0, 0, 0, []eni.NetworkInterface{crdv2}, types.EniSelectionPolicy(b.config.EniSelectionPolicy), nil)
+	mgr := eni.NewManager(0, 0, 0, 0, []eni.NetworkInterface{crdv2}, daemon.EniSelectionPolicy(b.config.EniSelectionPolicy), nil)
 
 	svc := b.RunENIMgr(b.ctx, mgr)
 	go b.service.startGarbageCollectionLoop(b.ctx)

daemon/config.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"github.com/AliyunContainerService/terway/pkg/aliyun/client"
-	"github.com/AliyunContainerService/terway/pkg/aliyun/instance"
 	"github.com/AliyunContainerService/terway/pkg/k8s"
 	"github.com/AliyunContainerService/terway/pkg/utils"
 	"github.com/AliyunContainerService/terway/pkg/vswitch"
@@ -25,26 +24,24 @@ func getDynamicConfig(ctx context.Context, k8s k8s.Kubernetes) (string, string,
 	return cfg, label, err
 }
 
-func getENIConfig(cfg *daemon.Config) *types.ENIConfig {
+func getENIConfig(cfg *daemon.Config) *daemon.ENIConfig {
 	vswitchSelectionPolicy := vswitch.VSwitchSelectionPolicyRandom
 	switch cfg.VSwitchSelectionPolicy {
 	case "ordered":
 		// keep the previous behave
 		vswitchSelectionPolicy = vswitch.VSwitchSelectionPolicyMost
 	}
 
-	eniSelectionPolicy := types.EniSelectionPolicyMostIPs
+	eniSelectionPolicy := daemon.EniSelectionPolicyMostIPs
 	switch cfg.EniSelectionPolicy {
 	case "least_ips":
-		eniSelectionPolicy = types.EniSelectionPolicyLeastIPs
+		eniSelectionPolicy = daemon.EniSelectionPolicyLeastIPs
 	}
 
-	eniConfig := &types.ENIConfig{
-		ZoneID:                 instance.GetInstanceMeta().ZoneID,
+	eniConfig := &daemon.ENIConfig{
 		VSwitchOptions:         nil,
 		ENITags:                cfg.ENITags,
 		SecurityGroupIDs:       cfg.GetSecurityGroups(),
-		InstanceID:             instance.GetInstanceMeta().InstanceID,
 		VSwitchSelectionPolicy: vswitchSelectionPolicy,
 		EniSelectionPolicy:     eniSelectionPolicy,
 		ResourceGroupID:        cfg.ResourceGroupID,
@@ -59,24 +56,20 @@ func getENIConfig(cfg *daemon.Config) *types.ENIConfig {
 		}
 	}
 
-	if len(eniConfig.VSwitchOptions) == 0 {
-		eniConfig.VSwitchOptions = []string{instance.GetInstanceMeta().VSwitchID}
-	}
-
 	if cfg.EnableENITrunking {
-		types.EnableFeature(&eniConfig.EniTypeAttr, types.FeatTrunk)
+		daemon.EnableFeature(&eniConfig.EniTypeAttr, daemon.FeatTrunk)
 	}
 	if cfg.EnableERDMA {
-		types.EnableFeature(&eniConfig.EniTypeAttr, types.FeatERDMA)
+		daemon.EnableFeature(&eniConfig.EniTypeAttr, daemon.FeatERDMA)
 	}
 
 	return eniConfig
 }
 
 // the actual size for pool is minIdle and maxIdle
-func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *client.Limits) (*types.PoolConfig, error) {
+func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *client.Limits) (*daemon.PoolConfig, error) {
 
-	poolConfig := &types.PoolConfig{
+	poolConfig := &daemon.PoolConfig{
 		BatchSize: 10,
 	}