Delay Gateway API CRD Removal Until After Cilium Update (#614)

Previously, Gateway API CRDs were installed and removed at the start of enabling or disabling the Gateway feature. This approach caused issues when the Gateway feature was disabled, as the CRDs were removed too early, leading to failures in Cilium's cleanup process due to the absence of these CRDs. This commit ensures that Gateway API CRDs are only removed after Cilium has been updated, preventing cleanup failures.
canonical · Aug 23, 2024 · a1dc288 · a1dc288
1 parent 51abfa3
commit a1dc288
Showing 1 changed file with 89 additions and 65 deletions.
diff --git a/src/k8s/pkg/k8sd/features/cilium/gateway.go b/src/k8s/pkg/k8sd/features/cilium/gateway.go
@@ -23,89 +23,113 @@ const (
 // ApplyGateway returns an error if anything fails. The error is also wrapped in the .Message field of the
 // returned FeatureStatus.
 func ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network, _ types.Annotations) (types.FeatureStatus, error) {
+	if gateway.GetEnabled() {
+		return enableGateway(ctx, snap)
+	}
+	return disableGateway(ctx, snap, network)
+}
+
+func enableGateway(ctx context.Context, snap snap.Snap) (types.FeatureStatus, error) {
 	m := snap.HelmClient()
 
-	if _, err := m.Apply(ctx, chartGateway, helm.StatePresentOrDeleted(gateway.GetEnabled()), nil); err != nil {
-		if gateway.GetEnabled() {
-			err = fmt.Errorf("failed to install Gateway API CRDs: %w", err)
-			return types.FeatureStatus{
-				Enabled: false,
-				Version: ciliumAgentImageTag,
-				Message: fmt.Sprintf(gatewayDeployFailedMsgTmpl, err),
-			}, err
-		} else {
-			err = fmt.Errorf("failed to delete Gateway API CRDs: %w", err)
-			return types.FeatureStatus{
-				Enabled: false,
-				Version: ciliumAgentImageTag,
-				Message: fmt.Sprintf(gatewayDeleteFailedMsgTmpl, err),
-			}, err
-		}
+	// Install Gateway API CRDs
+	if _, err := m.Apply(ctx, chartGateway, helm.StatePresent, nil); err != nil {
+		err = fmt.Errorf("failed to install Gateway API CRDs: %w", err)
+		return types.FeatureStatus{
+			Enabled: false,
+			Version: ciliumAgentImageTag,
+			Message: fmt.Sprintf(gatewayDeployFailedMsgTmpl, err),
+		}, err
 	}
 
 	// Apply our GatewayClass named ck-gateway
-	if _, err := m.Apply(ctx, chartGatewayClass, helm.StatePresentOrDeleted(gateway.GetEnabled()), nil); err != nil {
-		if gateway.GetEnabled() {
-			err = fmt.Errorf("failed to install Gateway API GatewayClass: %w", err)
-			return types.FeatureStatus{
-				Enabled: false,
-				Version: ciliumAgentImageTag,
-				Message: fmt.Sprintf(gatewayDeployFailedMsgTmpl, err),
-			}, err
-		} else {
-			err = fmt.Errorf("failed to delete Gateway API GatewayClass: %w", err)
-			return types.FeatureStatus{
-				Enabled: false,
-				Version: ciliumAgentImageTag,
-				Message: fmt.Sprintf(gatewayDeleteFailedMsgTmpl, err),
-			}, err
-		}
+	if _, err := m.Apply(ctx, chartGatewayClass, helm.StatePresent, nil); err != nil {
+		err = fmt.Errorf("failed to install Gateway API GatewayClass: %w", err)
+		return types.FeatureStatus{
+			Enabled: false,
+			Version: ciliumAgentImageTag,
+			Message: fmt.Sprintf(gatewayDeployFailedMsgTmpl, err),
+		}, err
 	}
 
-	changed, err := m.Apply(ctx, chartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), map[string]any{"gatewayAPI": map[string]any{"enabled": gateway.GetEnabled()}})
+	changed, err := m.Apply(ctx, chartCilium, helm.StateUpgradeOnly, map[string]any{"gatewayAPI": map[string]any{"enabled": true}})
 	if err != nil {
-		if gateway.GetEnabled() {
-			err = fmt.Errorf("failed to upgrade Gateway API cilium configuration: %w", err)
-			return types.FeatureStatus{
-				Enabled: false,
-				Version: ciliumAgentImageTag,
-				Message: fmt.Sprintf(gatewayDeployFailedMsgTmpl, err),
-			}, err
-		} else {
-			err = fmt.Errorf("failed to delete Gateway API cilium configuration: %w", err)
-			return types.FeatureStatus{
-				Enabled: false,
-				Version: ciliumAgentImageTag,
-				Message: fmt.Sprintf(gatewayDeleteFailedMsgTmpl, err),
-			}, err
-		}
+		err = fmt.Errorf("failed to upgrade Gateway API cilium configuration: %w", err)
+		return types.FeatureStatus{
+			Enabled: false,
+			Version: ciliumAgentImageTag,
+			Message: fmt.Sprintf(gatewayDeployFailedMsgTmpl, err),
+		}, err
 	}
 
 	if !changed {
-		if gateway.GetEnabled() {
-			return types.FeatureStatus{
-				Enabled: true,
-				Version: ciliumAgentImageTag,
-				Message: enabledMsg,
-			}, nil
-		} else {
-			return types.FeatureStatus{
-				Enabled: false,
-				Version: ciliumAgentImageTag,
-				Message: disabledMsg,
-			}, nil
-		}
+		return types.FeatureStatus{
+			Enabled: true,
+			Version: ciliumAgentImageTag,
+			Message: enabledMsg,
+		}, nil
 	}
 
-	if !gateway.GetEnabled() {
+	if err := rolloutRestartCilium(ctx, snap, 3); err != nil {
+		err = fmt.Errorf("failed to rollout restart cilium to enable Gateway API: %w", err)
+		return types.FeatureStatus{
+			Enabled: false,
+			Version: ciliumAgentImageTag,
+			Message: fmt.Sprintf(gatewayDeployFailedMsgTmpl, err),
+		}, err
+	}
+
+	return types.FeatureStatus{
+		Enabled: true,
+		Version: ciliumAgentImageTag,
+		Message: enabledMsg,
+	}, nil
+}
+
+func disableGateway(ctx context.Context, snap snap.Snap, network types.Network) (types.FeatureStatus, error) {
+	m := snap.HelmClient()
+
+	// Delete our GatewayClass named ck-gateway
+	if _, err := m.Apply(ctx, chartGatewayClass, helm.StateDeleted, nil); err != nil {
+		err = fmt.Errorf("failed to delete Gateway API GatewayClass: %w", err)
+		return types.FeatureStatus{
+			Enabled: false,
+			Version: ciliumAgentImageTag,
+			Message: fmt.Sprintf(gatewayDeleteFailedMsgTmpl, err),
+		}, err
+	}
+
+	changed, err := m.Apply(ctx, chartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), map[string]any{"gatewayAPI": map[string]any{"enabled": false}})
+	if err != nil {
+		err = fmt.Errorf("failed to delete Gateway API cilium configuration: %w", err)
+		return types.FeatureStatus{
+			Enabled: false,
+			Version: ciliumAgentImageTag,
+			Message: fmt.Sprintf(gatewayDeleteFailedMsgTmpl, err),
+		}, err
+	}
+
+	// Remove Gateway CRDs if the Gateway feature is disabled.
+	// This is done after the Cilium update as cilium requires the CRDs to be present for cleanups.
+	if _, err := m.Apply(ctx, chartGateway, helm.StateDeleted, nil); err != nil {
+		err = fmt.Errorf("failed to delete Gateway API CRDs: %w", err)
+		return types.FeatureStatus{
+			Enabled: false,
+			Version: ciliumAgentImageTag,
+			Message: fmt.Sprintf(gatewayDeleteFailedMsgTmpl, err),
+		}, err
+	}
+
+	if !changed {
 		return types.FeatureStatus{
 			Enabled: false,
 			Version: ciliumAgentImageTag,
 			Message: disabledMsg,
 		}, nil
 	}
+
 	if err := rolloutRestartCilium(ctx, snap, 3); err != nil {
-		err = fmt.Errorf("failed to rollout restart cilium to apply Gateway API: %w", err)
+		err = fmt.Errorf("failed to rollout restart cilium to disable Gateway API: %w", err)
 		return types.FeatureStatus{
 			Enabled: false,
 			Version: ciliumAgentImageTag,
@@ -114,8 +138,8 @@ func ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, ne
 	}
 
 	return types.FeatureStatus{
-		Enabled: true,
+		Enabled: false,
 		Version: ciliumAgentImageTag,
-		Message: enabledMsg,
+		Message: disabledMsg,
 	}, nil
 }