Skip to content

Commit

Permalink
🌱 Improve Machine remediation logs (kubernetes-sigs#11692)
Browse files Browse the repository at this point in the history
* Improve Machine remediation logs

* Address comments
  • Loading branch information
fabriziopandini authored Jan 20, 2025
1 parent 6979dbe commit 11ce72b
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -428,10 +428,11 @@ func (r *Reconciler) patchUnhealthyTargets(ctx context.Context, logger logr.Logg
// mark for remediation
errList := []error{}
for _, t := range unhealthy {
logger := logger.WithValues("Machine", klog.KObj(t.Machine), "Node", klog.KObj(t.Node))
condition := conditions.Get(t.Machine, clusterv1.MachineHealthCheckSucceededCondition)

if annotations.IsPaused(cluster, t.Machine) {
logger.Info("Machine has failed health check, but machine is paused so skipping remediation", "target", t.string(), "reason", condition.Reason, "message", condition.Message)
logger.Info("Machine has failed health check, but machine is paused so skipping remediation", "reason", condition.Reason, "message", condition.Message)
} else {
if m.Spec.RemediationTemplate != nil {
// If external remediation request already exists,
Expand Down Expand Up @@ -482,7 +483,7 @@ func (r *Reconciler) patchUnhealthyTargets(ctx context.Context, logger logr.Logg
// the same Machine, users are in charge of setting health checks and remediation properly.
to.SetName(t.Machine.Name)

logger.Info("Target has failed health check, creating an external remediation request", "remediation request name", to.GetName(), "target", t.string(), "reason", condition.Reason, "message", condition.Message)
logger.Info("Machine has failed health check, creating an external remediation request", "remediation request name", to.GetName(), "reason", condition.Reason, "message", condition.Message)
// Create the external clone.
if err := r.Client.Create(ctx, to); err != nil {
conditions.MarkFalse(m, clusterv1.ExternalRemediationRequestAvailableCondition, clusterv1.ExternalRemediationRequestCreationFailedReason, clusterv1.ConditionSeverityError, err.Error())
Expand All @@ -503,7 +504,7 @@ func (r *Reconciler) patchUnhealthyTargets(ctx context.Context, logger logr.Logg
Reason: clusterv1.MachineExternallyRemediatedWaitingForRemediationV1Beta2Reason,
})
} else if t.Machine.DeletionTimestamp.IsZero() { // Only setting the OwnerRemediated conditions when machine is not already in deletion.
logger.Info("Target has failed health check, marking for remediation", "target", t.string(), "reason", condition.Reason, "message", condition.Message)
logger.Info("Machine has failed health check, marking for remediation", "reason", condition.Reason, "message", condition.Message)
// NOTE: MHC is responsible for creating MachineOwnerRemediatedCondition if missing or to trigger another remediation if the previous one is completed;
// instead, if a remediation is in already progress, the remediation owner is responsible for completing the process and MHC should not overwrite the condition.
if !conditions.Has(t.Machine, clusterv1.MachineOwnerRemediatedCondition) || conditions.IsTrue(t.Machine, clusterv1.MachineOwnerRemediatedCondition) {
Expand Down Expand Up @@ -540,8 +541,9 @@ func (r *Reconciler) patchUnhealthyTargets(ctx context.Context, logger logr.Logg
t.Machine,
corev1.EventTypeNormal,
EventMachineMarkedUnhealthy,
"Machine %v has been marked as unhealthy",
t.string(),
"Machine %s has been marked as unhealthy by %s",
klog.KObj(t.Machine),
klog.KObj(t.MHC),
)
}
return errList
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,6 @@ type healthCheckTarget struct {
nodeMissing bool
}

func (t *healthCheckTarget) string() string {
return fmt.Sprintf("%s/%s/%s/%s",
t.MHC.GetNamespace(),
t.MHC.GetName(),
t.Machine.GetName(),
t.nodeName(),
)
}

// Get the node name if the target has a node.
func (t *healthCheckTarget) nodeName() string {
if t.Node != nil {
Expand Down Expand Up @@ -331,7 +322,7 @@ func (r *Reconciler) healthCheckTargets(targets []healthCheckTarget, logger logr
var healthy []healthCheckTarget

for _, t := range targets {
logger := logger.WithValues("target", t.string())
logger := logger.WithValues("Machine", klog.KObj(t.Machine), "Node", klog.KObj(t.Node))
logger.V(3).Info("Health checking target")
needsRemediation, nextCheck := t.needsRemediation(logger, timeoutForMachineToHaveNode)

Expand Down

0 comments on commit 11ce72b

Please sign in to comment.