All components support 2 deployment rolling (#142)

* rename QueryNodeController structs as DeployController Signed-off-by: shaoyue.chen <[email protected]> * Add deploy ctrl factory for future support of different component Signed-off-by: shaoyue.chen <[email protected]> * Add DeployModeChangerFactory to support different component Signed-off-by: shaoyue.chen <[email protected]> * Add DeployControllerBizUtilFactory to support different component Signed-off-by: shaoyue.chen <[email protected]> * changing mode annotation support all components Signed-off-by: shaoyue.chen <[email protected]> * Rename label querynode funcs Signed-off-by: shaoyue.chen <[email protected]> * K8sUtil support all components Signed-off-by: shaoyue.chen <[email protected]> * Fix tests Signed-off-by: shaoyue.chen <[email protected]> * group-id label support all components Signed-off-by: shaoyue.chen <[email protected]> * deploy_ctrl support all components Signed-off-by: shaoyue.chen <[email protected]> * Fix hpa Signed-off-by: shaoyue.chen <[email protected]> --------- Signed-off-by: shaoyue.chen <[email protected]>
zilliztech · Jul 1, 2024 · 1c918a5 · 1c918a5
1 parent e5f665d
commit 1c918a5
Show file tree

Hide file tree

Showing 32 changed files with 1,479 additions and 1,092 deletions.
diff --git a/apis/milvus.io/v1beta1/components_types.go b/apis/milvus.io/v1beta1/components_types.go
@@ -15,6 +15,17 @@ const (
 	QueryNode  ComponentType = "queryNode"
 	IndexNode  ComponentType = "indexNode"
 	Proxy      ComponentType = "proxy"
+
+	MixCoordName   = "mixcoord"
+	RootCoordName  = "rootcoord"
+	DataCoordName  = "datacoord"
+	QueryCoordName = "querycoord"
+	IndexCoordName = "indexcoord"
+	DataNodeName   = "datanode"
+	QueryNodeName  = "querynode"
+	IndexNodeName  = "indexnode"
+	ProxyName      = "proxy"
+	StandaloneName = "standalone"
 )
 
 var (
@@ -154,6 +165,10 @@ type MilvusComponents struct {
 	// +kubebuilder:validation:Optional
 	UpdateConfigMapOnly bool `json:"updateConfigMapOnly,omitempty"`
 
+	// RollingMode is the rolling mode for milvus components, default to 2
+	// +kubebuilder:validation:Optional
+	RollingMode RollingMode `json:"rollingMode,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	Proxy *MilvusProxy `json:"proxy,omitempty"`
 

diff --git a/apis/milvus.io/v1beta1/label_annotations.go b/apis/milvus.io/v1beta1/label_annotations.go
@@ -1,6 +1,7 @@
 package v1beta1
 
 import (
+	"fmt"
 	"strconv"
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -29,13 +30,6 @@ const (
 	PodServiceLabelAddedAnnotation = MilvusIO + "pod-service-label-added"
 	// ServiceLabel is the label to indicate whether the pod is a service pod
 	ServiceLabel = MilvusIO + "service"
-
-	// query node rolling related labels
-	MilvusIOLabelQueryNodeGroupId = MilvusIO + "querynode-group-id"
-	MilvusIOLabelQueryNodeRolling = MilvusIO + "querynode-rolling-id"
-	// query node rolling related annotations
-	MilvusIOAnnotationCurrentQueryNodeGroupId = MilvusIO + "current-querynode-group-id"
-	MilvusIOAnnotationChangingQueryNodeMode   = MilvusIO + "changing-querynode-mode"
 )
 
 // +kubebuilder:object:generate=false
@@ -47,65 +41,81 @@ func Labels() *LabelsImpl {
 	return singletonLabels
 }
 
-func (LabelsImpl) IsChangeQueryNodeMode(m Milvus) bool {
-	return m.Annotations[MilvusIOAnnotationChangingQueryNodeMode] == TrueStr
+func getChangingModeLabel(component string) string {
+	return fmt.Sprintf("%schanging-%s-mode", MilvusIO, component)
+}
+
+func GetComponentGroupIdLabel(component string) string {
+	return fmt.Sprintf("%s%s-group-id", MilvusIO, component)
+}
+
+func (LabelsImpl) IsChangingMode(m Milvus, component string) bool {
+	return m.Annotations[getChangingModeLabel(component)] == TrueStr
 }
 
-func (LabelsImpl) SetChangingQueryNodeMode(m *Milvus, changing bool) {
+func (LabelsImpl) SetChangingMode(m *Milvus, component string, changing bool) {
 	if changing {
-		m.Annotations[MilvusIOAnnotationChangingQueryNodeMode] = TrueStr
+		m.Annotations[getChangingModeLabel(component)] = TrueStr
 		return
 	}
-	delete(m.Annotations, MilvusIOAnnotationChangingQueryNodeMode)
+	delete(m.Annotations, getChangingModeLabel(component))
 }
 
-func (LabelsImpl) GetLabelQueryNodeGroupID(obj client.Object) string {
+func (LabelsImpl) GetLabelGroupID(component string, obj client.Object) string {
 	labels := obj.GetLabels()
 	if len(labels) < 1 {
 		return ""
 	}
-	return labels[MilvusIOLabelQueryNodeGroupId]
+	return labels[GetComponentGroupIdLabel(component)]
 }
 
-func (l LabelsImpl) SetQueryNodeGroupID(labels map[string]string, groupId int) {
-	l.SetQueryNodeGroupIDStr(labels, strconv.Itoa(groupId))
+func (l LabelsImpl) SetGroupID(component string, labels map[string]string, groupId int) {
+	l.SetGroupIDStr(component, labels, strconv.Itoa(groupId))
 }
 
-func (l LabelsImpl) SetQueryNodeGroupIDStr(labels map[string]string, groupIdStr string) {
-	labels[MilvusIOLabelQueryNodeGroupId] = groupIdStr
+func (l LabelsImpl) SetGroupIDStr(component string, labels map[string]string, groupIdStr string) {
+	labels[GetComponentGroupIdLabel(component)] = groupIdStr
 }
 
-func (LabelsImpl) GetCurrentQueryNodeGroupId(m *Milvus) string {
+func GetComponentCurrentGroupIDLabel(component string) string {
+	return fmt.Sprintf("%s%s-current-group-id", MilvusIO, component)
+}
+
+func (LabelsImpl) GetCurrentGroupId(m *Milvus, component string) string {
 	annot := m.GetAnnotations()
 	if len(annot) < 1 {
 		return ""
 	}
-	return annot[MilvusIOAnnotationCurrentQueryNodeGroupId]
+	return annot[GetComponentCurrentGroupIDLabel(component)]
+}
+
+func (l LabelsImpl) SetCurrentGroupID(m *Milvus, component string, groupId int) {
+	l.SetCurrentGroupIDStr(m, component, strconv.Itoa(groupId))
 }
 
-func (l LabelsImpl) SetCurrentQueryNodeGroupID(m *Milvus, groupId int) {
-	l.SetCurrentQueryNodeGroupIDStr(m, strconv.Itoa(groupId))
+func (LabelsImpl) SetCurrentGroupIDStr(m *Milvus, component string, groupId string) {
+	m.Annotations[GetComponentCurrentGroupIDLabel(component)] = groupId
 }
 
-func (LabelsImpl) SetCurrentQueryNodeGroupIDStr(m *Milvus, groupId string) {
-	m.Annotations[MilvusIOAnnotationCurrentQueryNodeGroupId] = groupId
+func GetRollingIdLabelByComponent(component string) string {
+	return fmt.Sprintf("%s%s-rolling-id", MilvusIO, component)
 }
 
-// IsQueryNodeRolling: if not empty, it means the query node has no rolling in progress
-func (LabelsImpl) IsQueryNodeRolling(m Milvus) bool {
-	return len(m.Labels[MilvusIOLabelQueryNodeRolling]) > 0
+// IsComponentRolling: if not empty, it means the component has no rolling in progress
+func (LabelsImpl) IsComponentRolling(m Milvus, component string) bool {
+	return len(m.Labels[GetRollingIdLabelByComponent(component)]) > 0
 }
 
-func (LabelsImpl) GetQueryNodeRollingId(m Milvus) string {
-	return m.Labels[MilvusIOLabelQueryNodeRolling]
+func (LabelsImpl) GetComponentRollingId(m Milvus, component string) string {
+	return m.Labels[GetRollingIdLabelByComponent(component)]
 }
 
-func (LabelsImpl) SetQueryNodeRolling(m *Milvus, rolling bool) {
+func (LabelsImpl) SetComponentRolling(m *Milvus, component string, rolling bool) {
 	if rolling {
-		if len(m.Labels[MilvusIOLabelQueryNodeRolling]) == 0 {
-			m.Labels[MilvusIOLabelQueryNodeRolling] = strconv.Itoa(int(m.GetGeneration()))
+		if len(m.Labels[GetRollingIdLabelByComponent(component)]) == 0 {
+			m.Labels[GetRollingIdLabelByComponent(component)] = strconv.Itoa(int(m.GetGeneration()))
 		}
 		return
 	}
-	delete(m.Labels, MilvusIOLabelQueryNodeRolling)
+	delete(m.Labels, GetRollingIdLabelByComponent(component))
 }
diff --git a/apis/milvus.io/v1beta1/label_annotations_test.go b/apis/milvus.io/v1beta1/label_annotations_test.go
@@ -9,44 +9,44 @@ import (
 func TestLabelsImpl_IsChangeQueryNodeMode_SetChangingQueryNodeMode(t *testing.T) {
 	mc := Milvus{}
 	mc.Default()
-	assert.False(t, Labels().IsChangeQueryNodeMode(mc))
-	Labels().SetChangingQueryNodeMode(&mc, true)
-	assert.True(t, Labels().IsChangeQueryNodeMode(mc))
-	Labels().SetChangingQueryNodeMode(&mc, false)
-	assert.False(t, Labels().IsChangeQueryNodeMode(mc))
+	assert.False(t, Labels().IsChangingMode(mc, DataNodeName))
+	Labels().SetChangingMode(&mc, DataNodeName, true)
+	assert.True(t, Labels().IsChangingMode(mc, DataNodeName))
+	Labels().SetChangingMode(&mc, DataNodeName, false)
+	assert.False(t, Labels().IsChangingMode(mc, DataNodeName))
 }
 
 func TestLabelsImpl_GetLabelQueryNodeGroupID_SetQueryNodeGroupID(t *testing.T) {
 	mc := Milvus{}
 	mc.Default()
-	assert.Equal(t, "", Labels().GetLabelQueryNodeGroupID(&mc))
-	Labels().SetQueryNodeGroupID(mc.Labels, 1)
-	assert.Equal(t, "1", Labels().GetLabelQueryNodeGroupID(&mc))
-	Labels().SetQueryNodeGroupID(mc.Labels, 0)
-	assert.Equal(t, "0", Labels().GetLabelQueryNodeGroupID(&mc))
+	assert.Equal(t, "", Labels().GetLabelGroupID(DataNodeName, &mc))
+	Labels().SetGroupID(DataNodeName, mc.Labels, 1)
+	assert.Equal(t, "1", Labels().GetLabelGroupID(DataNodeName, &mc))
+	Labels().SetGroupID(DataNodeName, mc.Labels, 0)
+	assert.Equal(t, "0", Labels().GetLabelGroupID(DataNodeName, &mc))
 
 }
 
 func TestLabelsImpl_GetCurrentQueryNodeGroupId_SetCurrentQueryNodeGroupID(t *testing.T) {
 	mc := Milvus{}
 	mc.Default()
-	assert.Equal(t, "", Labels().GetCurrentQueryNodeGroupId(&mc))
-	Labels().SetCurrentQueryNodeGroupID(&mc, 1)
-	assert.Equal(t, "1", Labels().GetCurrentQueryNodeGroupId(&mc))
-	Labels().SetCurrentQueryNodeGroupID(&mc, 0)
-	assert.Equal(t, "0", Labels().GetCurrentQueryNodeGroupId(&mc))
+	assert.Equal(t, "", Labels().GetCurrentGroupId(&mc, DataNodeName))
+	Labels().SetCurrentGroupID(&mc, DataNodeName, 1)
+	assert.Equal(t, "1", Labels().GetCurrentGroupId(&mc, DataNodeName))
+	Labels().SetCurrentGroupID(&mc, DataNodeName, 0)
+	assert.Equal(t, "0", Labels().GetCurrentGroupId(&mc, DataNodeName))
 }
 
 func TestLabelsImpl_IsQueryNodeRolling_GetQueryNodeRollingId_SetQueryNodeRolling(t *testing.T) {
 	mc := Milvus{}
 	mc.Generation = 1
 	mc.Default()
-	assert.False(t, Labels().IsQueryNodeRolling(mc))
-	assert.Equal(t, "", Labels().GetQueryNodeRollingId(mc))
-	Labels().SetQueryNodeRolling(&mc, true)
-	assert.True(t, Labels().IsQueryNodeRolling(mc))
-	assert.Equal(t, "1", Labels().GetQueryNodeRollingId(mc))
-	Labels().SetQueryNodeRolling(&mc, false)
-	assert.False(t, Labels().IsQueryNodeRolling(mc))
-	assert.Equal(t, "", Labels().GetQueryNodeRollingId(mc))
+	assert.False(t, Labels().IsComponentRolling(mc, DataNodeName))
+	assert.Equal(t, "", Labels().GetComponentRollingId(mc, DataNodeName))
+	Labels().SetComponentRolling(&mc, DataNodeName, true)
+	assert.True(t, Labels().IsComponentRolling(mc, DataNodeName))
+	assert.Equal(t, "1", Labels().GetComponentRollingId(mc, DataNodeName))
+	Labels().SetComponentRolling(&mc, DataNodeName, false)
+	assert.False(t, Labels().IsComponentRolling(mc, DataNodeName))
+	assert.Equal(t, "", Labels().GetComponentRollingId(mc, DataNodeName))
 }
diff --git a/apis/milvus.io/v1beta1/milvus_types.go b/apis/milvus.io/v1beta1/milvus_types.go
@@ -183,11 +183,25 @@ type RollingMode int
 
 // RollingMode definitions
 const (
+	// RollingModeNotSet is only used before init
 	RollingModeNotSet RollingMode = iota
-	// this mode has 1 query node deployment, uses k8s deployment's default rolling update strategy
+	// deprecated, shall not be used spec: in this mode all components are in OneDeployMode
 	RollingModeV1
-	// this mode has 2 query node deployment, operator takes care of querynode rolling update
+	// in this mode only querynode is in TwoDeployMode, other components are in OneDeployMode
 	RollingModeV2
+	// in this mode all components are in TwoDeployMode
+	RollingModeV3
+)
+
+type ComponentDeployMode int
+
+const (
+	// DeployModeUnknown is used when the component's deploy mode is unknown
+	DeployModeUnknown ComponentDeployMode = iota
+	// component in this mode has 1 deployment, uses k8s deployment's default rolling update strategy
+	OneDeployMode
+	// component in this mode has 2 deployment, operator takes care of querynode rolling update
+	TwoDeployMode
 )
 
 type ComponentDeployStatus struct {

diff --git a/apis/milvus.io/v1beta1/milvus_webhook.go b/apis/milvus.io/v1beta1/milvus_webhook.go
@@ -82,6 +82,12 @@ func (r *Milvus) ValidateCreate() error {
 }
 
 func (r *Milvus) validateCommon() *field.Error {
+	switch r.Spec.Com.RollingMode {
+	case RollingModeNotSet, RollingModeV2, RollingModeV3:
+	default:
+		fp := field.NewPath("spec").Child("components").Child("rollingMode")
+		return field.Invalid(fp, r.Spec.Com.RollingMode, "rollingMode should be 2 or 3")
+	}
 	if err := r.validateEnableRolingUpdate(); err != nil {
 		return err
 	}
@@ -240,6 +246,9 @@ func (r *Milvus) DefaultComponents() {
 	if spec.Com.ImageUpdateMode == "" {
 		spec.Com.ImageUpdateMode = ImageUpdateModeRollingUpgrade
 	}
+	if spec.Com.RollingMode == RollingModeNotSet {
+		spec.Com.RollingMode = RollingModeV2
+	}
 	if spec.Com.Standalone == nil {
 		spec.Com.Standalone = &MilvusStandalone{}
 	}

diff --git a/apis/milvus.io/v1beta1/milvus_webhook_test.go b/apis/milvus.io/v1beta1/milvus_webhook_test.go
@@ -60,6 +60,7 @@ func TestMilvus_Default_NotExternal(t *testing.T) {
 					Component: defaultComponent,
 				},
 			},
+			RollingMode: RollingModeV2,
 		},
 		Conf: Values{
 			Data: map[string]interface{}{},
@@ -98,6 +99,7 @@ func TestMilvus_Default_NotExternal(t *testing.T) {
 		ComponentSpec: ComponentSpec{
 			Image: config.DefaultMilvusImage,
 		},
+		RollingMode: RollingModeV2,
 		Proxy: &MilvusProxy{
 			ServiceComponent: ServiceComponent{
 				Component: defaultComponent,
@@ -219,11 +221,8 @@ func TestMilvus_Default_DeleteUnSetableOK(t *testing.T) {
 
 func TestMilvus_ValidateCreate_NoError(t *testing.T) {
 	mc := Milvus{}
-	err := mc.ValidateCreate()
-	assert.NoError(t, err)
-
 	mc.Default()
-	err = mc.ValidateCreate()
+	err := mc.ValidateCreate()
 	assert.NoError(t, err)
 }
 
@@ -257,6 +256,7 @@ func TestMilvus_ValidateCreate_Invalid3(t *testing.T) {
 			},
 		},
 	}
+	mc.Default()
 	err := mc.ValidateCreate()
 	assert.Error(t, err)
 }
@@ -339,6 +339,15 @@ func Test_DefaultConf_EnableRollingUpdate(t *testing.T) {
 
 func TestMilvus_validateCommon(t *testing.T) {
 	mc := Milvus{}
+	t.Run("rolling mode <2 or >3 not support", func(t *testing.T) {
+		mc.Spec.Com.RollingMode = RollingModeV1
+		err := mc.validateCommon()
+		assert.Error(t, err)
+		mc.Spec.Com.RollingMode = 4
+		err = mc.validateCommon()
+		assert.Error(t, err)
+	})
+	mc.Spec.Com.RollingMode = RollingModeV2
 	t.Run("validate rollingupdate", func(t *testing.T) {
 		mc.Spec.Com.EnableRollingUpdate = util.BoolPtr(true)
 		err := mc.validateCommon()

diff --git a/charts/milvus-operator/templates/crds.yaml b/charts/milvus-operator/templates/crds.yaml
@@ -4853,6 +4853,8 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  rollingMode:
+                    type: integer
                   rootCoord:
                     properties:
                       affinity:
@@ -12166,6 +12168,8 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  rollingMode:
+                    type: integer
                   rootCoord:
                     properties:
                       affinity:

diff --git a/config/crd/bases/milvus.io_milvusclusters.yaml b/config/crd/bases/milvus.io_milvusclusters.yaml
@@ -4852,6 +4852,8 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  rollingMode:
+                    type: integer
                   rootCoord:
                     properties:
                       affinity:

diff --git a/config/crd/bases/milvus.io_milvuses.yaml b/config/crd/bases/milvus.io_milvuses.yaml
@@ -5794,6 +5794,8 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  rollingMode:
+                    type: integer
                   rootCoord:
                     properties:
                       affinity:

diff --git a/deploy/manifests/deployment.yaml b/deploy/manifests/deployment.yaml
@@ -4884,6 +4884,8 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  rollingMode:
+                    type: integer
                   rootCoord:
                     properties:
                       affinity:
@@ -12198,6 +12200,8 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  rollingMode:
+                    type: integer
                   rootCoord:
                     properties:
                       affinity: