K8SPXC-1534 | [bug] fix issue with inconsistent secret reconciliation

pkg/controller/pxc/controller.go

@@ -19,6 +19,7 @@ import (
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -42,6 +43,10 @@ import (
 	"github.com/percona/percona-xtradb-cluster-operator/version"
 )
 
+const (
+	secretsNameField = ".spec.secretsName"
+)
+
 // Add creates a new PerconaXtraDBCluster Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
 func Add(mgr manager.Manager) error {
@@ -77,12 +82,52 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
 
 // add adds a new Controller to mgr with r as the reconcile.Reconciler
 func add(mgr manager.Manager, r reconcile.Reconciler) error {
+	if err := setupFieldIndexers(mgr); err != nil {
+		return errors.Wrap(err, "setup field indexers")
+	}
 	return builder.ControllerManagedBy(mgr).
 		Named("pxc-controller").
 		Watches(&api.PerconaXtraDBCluster{}, &handler.EnqueueRequestForObject{}).
+		Watches(&corev1.Secret{}, enqueuePXCReferencingSecret(mgr.GetClient())).
 		Complete(r)
 }
 
+func setupFieldIndexers(mgr manager.Manager) error {
+	return mgr.GetFieldIndexer().IndexField(context.TODO(), &api.PerconaXtraDBCluster{}, secretsNameField, func(o client.Object) []string {
+		cluster, ok := o.(*api.PerconaXtraDBCluster)
+		if !ok {
+			return nil
+		}
+		return []string{cluster.Spec.SecretsName}
+	})
+}
+
+// enqueuePXCReferencingSecret returns an EventHandler that returns a list of all
+// pxc-clusters that reference the secret via `.spec.secretsName`.
+func enqueuePXCReferencingSecret(c client.Client) handler.EventHandler {
+	return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request {
+		secret, ok := o.(*corev1.Secret)
+		if !ok {
+			return nil
+		}
+		list := &api.PerconaXtraDBClusterList{}
+		_ = c.List(context.TODO(), list, &client.ListOptions{
+			FieldSelector: fields.OneTermEqualSelector(secretsNameField, secret.GetName()),
+			Namespace:     secret.GetNamespace(),
+		})
+		var requests []reconcile.Request
+		for _, cr := range list.Items {
+			requests = append(requests, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Name:      cr.GetName(),
+					Namespace: cr.GetNamespace(),
+				},
+			})
+		}
+		return requests
+	})
+}
+
 var _ reconcile.Reconciler = &ReconcilePerconaXtraDBCluster{}
 
 // ReconcilePerconaXtraDBCluster reconciles a PerconaXtraDBCluster object
@@ -296,7 +341,7 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(ctx context.Context, request r
 		}
 	}
 
-	err = r.reconcileUsersSecret(ctx, o)
+	userSecret, err := r.reconcileUsersSecret(ctx, o)
 	if err != nil {
 		return reconcile.Result{}, errors.Wrap(err, "reconcile users secret")
 	}
@@ -305,7 +350,7 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(ctx context.Context, request r
 	// Currently, if an error occurs before the statefulsets are updated with annotations, and reconcileUsers has a different result on the next reconcile, the statefulsets will not have the required annotations.
 	userReconcileResult := &ReconcileUsersResult{}
 
-	urr, err := r.reconcileUsers(ctx, o)
+	urr, err := r.reconcileUsers(ctx, o, userSecret)
 	if err != nil {
 		return reconcile.Result{}, errors.Wrap(err, "reconcile users")
 	}

pkg/controller/pxc/secrets.go

@@ -23,11 +23,12 @@ import (
 
 const internalSecretsPrefix = "internal-"
 
-func (r *ReconcilePerconaXtraDBCluster) reconcileUsersSecret(ctx context.Context, cr *api.PerconaXtraDBCluster) error {
+// reconciles the user secret provided in `.spec.secretName` field, and returns the updated/created secret.
+func (r *ReconcilePerconaXtraDBCluster) reconcileUsersSecret(ctx context.Context, cr *api.PerconaXtraDBCluster) (*corev1.Secret, error) {
 	log := logf.FromContext(ctx)
 
 	secretObj := new(corev1.Secret)
-	err := r.client.Get(context.TODO(),
+	err := r.client.Get(ctx,
 		types.NamespacedName{
 			Namespace: cr.Namespace,
 			Name:      cr.Spec.SecretsName,
@@ -36,22 +37,22 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileUsersSecret(ctx context.Context
 	)
 	if err == nil {
 		if err := validatePasswords(secretObj); err != nil {
-			return errors.Wrap(err, "validate passwords")
+			return nil, errors.Wrap(err, "validate passwords")
 		}
 		isChanged, err := setUserSecretDefaults(secretObj)
 		if err != nil {
-			return errors.Wrap(err, "set user secret defaults")
+			return nil, errors.Wrap(err, "set user secret defaults")
 		}
 		if isChanged {
 			err := r.client.Update(context.TODO(), secretObj)
 			if err == nil {
 				log.Info("User secrets updated", "secrets", cr.Spec.SecretsName)
 			}
-			return err
+			return secretObj, err
 		}
-		return nil
+		return secretObj, nil
 	} else if !k8serror.IsNotFound(err) {
-		return errors.Wrap(err, "get secret")
+		return nil, errors.Wrap(err, "get secret")
 	}
 
 	secretObj = &corev1.Secret{
@@ -64,16 +65,16 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileUsersSecret(ctx context.Context
 	}
 
 	if _, err = setUserSecretDefaults(secretObj); err != nil {
-		return errors.Wrap(err, "set user secret defaults")
+		return nil, errors.Wrap(err, "set user secret defaults")
 	}
 
 	err = r.client.Create(context.TODO(), secretObj)
 	if err != nil {
-		return fmt.Errorf("create Users secret: %v", err)
+		return nil, fmt.Errorf("create Users secret: %v", err)
 	}
 
 	log.Info("Created user secrets", "secrets", cr.Spec.SecretsName)
-	return nil
+	return secretObj, nil
 }
 
 func setUserSecretDefaults(secret *corev1.Secret) (isChanged bool, err error) {

pkg/controller/pxc/users.go

@@ -46,27 +46,16 @@ type ReconcileUsersResult struct {
 	updateReplicationPassword bool
 }
 
-func (r *ReconcilePerconaXtraDBCluster) reconcileUsers(ctx context.Context, cr *api.PerconaXtraDBCluster) (*ReconcileUsersResult, error) {
+func (r *ReconcilePerconaXtraDBCluster) reconcileUsers(
+	ctx context.Context,
+	cr *api.PerconaXtraDBCluster,
+	secrets *corev1.Secret,
+) (*ReconcileUsersResult, error) {
 	log := logf.FromContext(ctx)
 
-	secrets := corev1.Secret{}
-	err := r.client.Get(context.TODO(),
-		types.NamespacedName{
-			Namespace: cr.Namespace,
-			Name:      cr.Spec.SecretsName,
-		},
-		&secrets,
-	)
-	if err != nil && k8serrors.IsNotFound(err) {
-		return nil, nil
-	} else if err != nil {
-		return nil, errors.Wrapf(err, "get sys users secret '%s'", cr.Spec.SecretsName)
-	}
-
 	internalSecretName := internalSecretsPrefix + cr.Name
-
 	internalSecrets := corev1.Secret{}
-	err = r.client.Get(context.TODO(),
+	err := r.client.Get(context.TODO(),
 		types.NamespacedName{
 			Namespace: cr.Namespace,
 			Name:      internalSecretName,
@@ -109,12 +98,12 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileUsers(ctx context.Context, cr *
 
 	var actions *userUpdateActions
 	if ver.GreaterThanOrEqual(mysql80) {
-		actions, err = r.updateUsers(ctx, cr, &secrets, &internalSecrets)
+		actions, err = r.updateUsers(ctx, cr, secrets, &internalSecrets)
 		if err != nil {
 			return nil, errors.Wrap(err, "manage sys users")
 		}
 	} else {
-		actions, err = r.updateUsersWithoutDP(ctx, cr, &secrets, &internalSecrets)
+		actions, err = r.updateUsersWithoutDP(ctx, cr, secrets, &internalSecrets)
 		if err != nil {
 			return nil, errors.Wrap(err, "manage sys users")
 		}