Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PTEUDO-1422: Publish status events for state changes in database #377

Merged
merged 11 commits into from
Dec 19, 2024
174 changes: 85 additions & 89 deletions pkg/databaseclaim/claimstatus.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,23 @@ import (
"time"

v1 "github.com/infobloxopen/db-controller/api/v1"
basefun "github.com/infobloxopen/db-controller/pkg/basefunctions"
"github.com/infobloxopen/db-controller/pkg/hostparams"
"github.com/infobloxopen/db-controller/pkg/pgctl"
"github.com/spf13/viper"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"

"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

var (
ErrDoNotUpdateStatus = fmt.Errorf("do not update status for this error")
)

func (r *DatabaseClaimReconciler) manageError(ctx context.Context, dbClaim *v1.DatabaseClaim, inErr error) (ctrl.Result, error) {
// Class of errors that should stop the reconciliation loop
// but not cause a status change on the CR
if errors.Is(inErr, ErrDoNotUpdateStatus) {
return ctrl.Result{}, nil
}
return manageError(ctx, r.Client, dbClaim, inErr)
}

type managedErr struct {
err error
}
Expand All @@ -36,58 +32,80 @@ func (m *managedErr) Error() string {
return m.err.Error()
}

func refreshClaim(ctx context.Context, cli client.Client, claim *v1.DatabaseClaim) error {
nname := types.NamespacedName{
Namespace: claim.Namespace,
Name: claim.Name,
}
type StatusManager struct {
client client.Client
passwordRotationTime time.Duration
}

logr := log.FromContext(ctx).WithValues("databaseclaim", nname)
if err := cli.Get(ctx, nname, claim); err != nil {
logr.Error(err, "refresh_claim")
return err
}
func NewStatusManager(c client.Client, viper *viper.Viper) *StatusManager {
return &StatusManager{client: c, passwordRotationTime: basefun.GetPasswordRotationPeriod(viper)}
}

return nil
func (m *StatusManager) UpdateStatus(ctx context.Context, dbClaim *v1.DatabaseClaim) error {
return m.client.Status().Update(ctx, dbClaim)
}

// manageError updates the status of the DatabaseClaim to
// reflect the error that occurred. It returns the error that
// should be returned by the Reconcile function.
func manageError(ctx context.Context, cli client.Client, claim *v1.DatabaseClaim, inErr error) (ctrl.Result, error) {
func (m *StatusManager) SetErrorStatus(ctx context.Context, dbClaim *v1.DatabaseClaim, inErr error) (reconcile.Result, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can change the method name to SetError since the struct already has the Status (StatusManager).

// If the error is non-critical and doesn't require a status update, skip processing
if errors.Is(inErr, ErrDoNotUpdateStatus) {
return ctrl.Result{}, nil
}

nname := types.NamespacedName{
Namespace: claim.Namespace,
Name: claim.Name,
Namespace: dbClaim.Namespace,
Name: dbClaim.Name,
}
logr := log.FromContext(ctx).WithValues("databaseclaim", nname)

// Prevent manageError being called multiple times on the same error
wrappedErr, ok := inErr.(*managedErr)
if ok {
logr.Error(inErr, fmt.Sprintf("manageError called multiple times on the same error: %#v", inErr))
m.SetStatusCondition(ctx, dbClaim, ReconcileErrorCondition(inErr))
var wrappedErr *managedErr
if existingErr, isManaged := inErr.(*managedErr); isManaged {
logr.Error(existingErr, "manageError called multiple times for the same error")
Copy link
Contributor

@drewwells drewwells Dec 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At some point, this could be promoted to a panic. So we can identify and fix the duplidate error calls happening

wrappedErr = existingErr
} else {
wrappedErr = &managedErr{err: inErr}
}

// Refresh the claim to get the latest resource version
copy := claim.DeepCopy()
if err := refreshClaim(ctx, cli, copy); err != nil {
logr.Error(err, "manager_error_refresh_claim")
refreshedClaim := dbClaim.DeepCopy()
if err := m.client.Get(ctx, nname, refreshedClaim); err != nil {
logr.Error(err, "Failed to refresh DatabaseClaim")
return ctrl.Result{}, wrappedErr
}

copy.Status.Error = inErr.Error()
if err := cli.Status().Update(ctx, copy); err != nil {
logr.Error(err, "manager_error_update_claim")
refreshedClaim.Status.Error = wrappedErr.Error()
if err := m.UpdateStatus(ctx, dbClaim); err != nil {
logr.Error(err, "Failed to update DatabaseClaim status")
return ctrl.Result{}, wrappedErr
}

logr.Info("DatabaseClaim status updated with error", "error", wrappedErr.Error())
return ctrl.Result{}, wrappedErr
}

func SetDBClaimCondition(ctx context.Context, dbClaim *v1.DatabaseClaim, condition metav1.Condition) {
logf := log.FromContext(ctx)
func (m *StatusManager) SuccessAndUpdateCondition(ctx context.Context, dbClaim *v1.DatabaseClaim) (reconcile.Result, error) {
logf := log.FromContext(ctx).WithValues("databaseclaim", dbClaim.Name)

dbClaim.Status.Error = ""
m.SetStatusCondition(ctx, dbClaim, ReconcileSuccessCondition())
if err := m.UpdateStatus(ctx, dbClaim); err != nil {
logf.Error(err, "Error updating DatabaseClaim status")
return ctrl.Result{}, err
}

//if object is getting deleted then call requeue immediately
if !dbClaim.ObjectMeta.DeletionTimestamp.IsZero() {
return ctrl.Result{Requeue: true}, nil
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't you check for deleted state before updating the status?


if dbClaim.Status.OldDB.DbState == v1.PostMigrationInProgress {
return ctrl.Result{RequeueAfter: time.Minute}, nil
}

return ctrl.Result{RequeueAfter: m.passwordRotationTime}, nil
}

func (m *StatusManager) SetStatusCondition(ctx context.Context, dbClaim *v1.DatabaseClaim, condition metav1.Condition) {
logf := log.FromContext(ctx).WithValues("databaseclaim", dbClaim.Name)

condition.LastTransitionTime = metav1.Now()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The LastTransitionTime is set already when you create the condition:

func CreateCondition(condType ConditionType, status metav1.ConditionStatus, reason, message string) metav1.Condition {
now := metav1.NewTime(time.Now())
return metav1.Condition{
Type: string(condType),
Status: status,
Reason: reason,
Message: message,
LastTransitionTime: now,
}
}
.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed from the create condition as it makes more sense to keep it here, closer to the update operation.

condition.ObservedGeneration = dbClaim.Generation
Expand All @@ -98,7 +116,7 @@ func SetDBClaimCondition(ctx context.Context, dbClaim *v1.DatabaseClaim, conditi
if condition.Status == cond.Status && !condition.LastTransitionTime.IsZero() {
condition.LastTransitionTime = cond.LastTransitionTime
} else {
logf.V(1).Info("Condition status changed %s -> %s", cond.Status, condition.Status)
logf.V(1).Info("Condition status changed %s -> %s", string(cond.Status), string(condition.Status))
}
dbClaim.Status.Conditions[idx] = condition
return
Expand All @@ -108,50 +126,17 @@ func SetDBClaimCondition(ctx context.Context, dbClaim *v1.DatabaseClaim, conditi
dbClaim.Status.Conditions = append(dbClaim.Status.Conditions, condition)
}

func (r *DatabaseClaimReconciler) manageSuccess(ctx context.Context, dbClaim *v1.DatabaseClaim) (ctrl.Result, error) {

dbClaim.Status.Error = ""
SetDBClaimCondition(ctx, dbClaim, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionTrue,
Reason: "Ready",
Message: "databaseclaim is up to date and ready",
})

if err := r.Client.Status().Update(ctx, dbClaim); err != nil {
logr := log.FromContext(ctx).WithValues("databaseclaim", dbClaim.Name)
logr.Error(err, "manage_success_update_claim")
return ctrl.Result{}, err
}
//if object is getting deleted then call requeue immediately
if !dbClaim.ObjectMeta.DeletionTimestamp.IsZero() {
return ctrl.Result{Requeue: true}, nil
}
if dbClaim.Status.OldDB.DbState == v1.PostMigrationInProgress {
return ctrl.Result{RequeueAfter: time.Minute}, nil
}

return ctrl.Result{RequeueAfter: r.getPasswordRotationTime()}, nil
}

func (r *DatabaseClaimReconciler) updateClientStatus(ctx context.Context, dbClaim *v1.DatabaseClaim) error {

return r.Client.Status().Update(ctx, dbClaim)
}

func updateUserStatus(status *v1.Status, reqInfo *requestInfo, userName, userPassword string) {
timeNow := metav1.Now()
if status.ConnectionInfo == nil {
status.ConnectionInfo = &v1.DatabaseClaimConnectionInfo{}
func (m *StatusManager) UpdateClusterStatus(status *v1.Status, hostParams *hostparams.HostParams) {
status.DBVersion = hostParams.DBVersion
status.Type = v1.DatabaseType(hostParams.Type)
status.Shape = hostParams.Shape
status.MinStorageGB = hostParams.MinStorageGB
if hostParams.Type == string(v1.Postgres) {
status.MaxStorageGB = hostParams.MaxStorageGB
}

status.UserUpdatedAt = &timeNow
status.ConnectionInfo.Username = userName
reqInfo.TempSecret = userPassword
status.ConnectionInfoUpdatedAt = &timeNow
}

func updateDBStatus(status *v1.Status, dbName string) {
func (m *StatusManager) UpdateDBStatus(status *v1.Status, dbName string) {
timeNow := metav1.Now()
if status.DbCreatedAt == nil {
status.DbCreatedAt = &timeNow
Expand All @@ -165,7 +150,7 @@ func updateDBStatus(status *v1.Status, dbName string) {
}
}

func updateHostPortStatus(status *v1.Status, host, port, sslMode string) {
func (m *StatusManager) UpdateHostPortStatus(status *v1.Status, host string, port string, sslMode string) {
timeNow := metav1.Now()
if status.ConnectionInfo == nil {
status.ConnectionInfo = &v1.DatabaseClaimConnectionInfo{}
Expand All @@ -176,12 +161,23 @@ func updateHostPortStatus(status *v1.Status, host, port, sslMode string) {
status.ConnectionInfoUpdatedAt = &timeNow
}

func updateClusterStatus(status *v1.Status, hostParams *hostparams.HostParams) {
status.DBVersion = hostParams.DBVersion
status.Type = v1.DatabaseType(hostParams.Type)
status.Shape = hostParams.Shape
status.MinStorageGB = hostParams.MinStorageGB
if hostParams.Type == string(v1.Postgres) {
status.MaxStorageGB = hostParams.MaxStorageGB
func (m *StatusManager) UpdateUserStatus(status *v1.Status, reqInfo *requestInfo, userName string, userPassword string) {
timeNow := metav1.Now()
if status.ConnectionInfo == nil {
status.ConnectionInfo = &v1.DatabaseClaimConnectionInfo{}
}

status.UserUpdatedAt = &timeNow
status.ConnectionInfo.Username = userName
reqInfo.TempSecret = userPassword
status.ConnectionInfoUpdatedAt = &timeNow
}

func (m *StatusManager) MigrationInProgressStatus(ctx context.Context, dbClaim *v1.DatabaseClaim) (reconcile.Result, error) {
dbClaim.Status.MigrationState = pgctl.S_MigrationInProgress.String()

m.SetStatusCondition(ctx, dbClaim, MigratingCondition())

err := m.UpdateStatus(ctx, dbClaim)
return ctrl.Result{Requeue: true}, err
}
Loading
Loading