Skip to content

Commit

Permalink
Merge pull request #1347 from petrkotas/aro-operator-rh-key-condition
Browse files Browse the repository at this point in the history 
Add ARO operator status condition with Red Hat key presence info
  • Loading branch information
Mangirdas Judeikis authored Mar 22, 2021
2 parents 0e70074 + 4e41e3e commit 2fac92d
Show file tree
Hide file tree
Showing 6 changed files with 1,116 additions and 89 deletions.
2 changes: 1 addition & 1 deletion cmd/aro/operator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func operator(ctx context.Context, log *logrus.Entry) error {
}
if err = (pullsecret.NewReconciler(
log.WithField("controller", controllers.PullSecretControllerName),
kubernetescli)).SetupWithManager(mgr); err != nil {
kubernetescli, arocli)).SetupWithManager(mgr); err != nil {
return fmt.Errorf("unable to create controller PullSecret: %v", err)
}
if err = (alertwebhook.NewReconciler(
Expand Down
3 changes: 2 additions & 1 deletion pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,13 @@ const (
InternetReachableFromMaster status.ConditionType = "InternetReachableFromMaster"
InternetReachableFromWorker status.ConditionType = "InternetReachableFromWorker"
MachineValid status.ConditionType = "MachineValid"
RedHatKeyPresent status.ConditionType = "RedHatKeyPresent"
)

// AllConditionTypes is a operator conditions currently in use, any condition not in this list is not
// added to the operator.status.conditions list
func AllConditionTypes() []status.ConditionType {
return []status.ConditionType{InternetReachableFromMaster, InternetReachableFromWorker, MachineValid}
return []status.ConditionType{InternetReachableFromMaster, InternetReachableFromWorker, MachineValid, RedHatKeyPresent}
}

// ClusterChecksTypes represents checks performed on the cluster to verify basic functionality
Expand Down
241 changes: 168 additions & 73 deletions pkg/operator/controllers/pullsecret/pullsecret_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,19 @@ package pullsecret
// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.

// Image Registry pull-secret reconciler
// Users tend to do damage to corev1.Secret openshift-config/pull-secret
// this controllers ensures valid ARO secret for Azure mirror with
// openshift images
// It also signals presense of Red Hat image registry keys via operator conditions

import (
"context"
"encoding/json"
"errors"
"strings"

"github.com/operator-framework/operator-sdk/pkg/status"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -24,22 +33,26 @@ import (

"github.com/Azure/ARO-RP/pkg/operator"
arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
"github.com/Azure/ARO-RP/pkg/operator/controllers"
"github.com/Azure/ARO-RP/pkg/util/pullsecret"
)

var pullSecretName = types.NamespacedName{Name: "pull-secret", Namespace: "openshift-config"}
var rhKeys = []string{"registry.redhat.io", "cloud.redhat.com"}

// PullSecretReconciler reconciles a Cluster object
type PullSecretReconciler struct {
kubernetescli kubernetes.Interface
arocli aroclient.Interface
log *logrus.Entry
}

func NewReconciler(log *logrus.Entry, kubernetescli kubernetes.Interface) *PullSecretReconciler {
func NewReconciler(log *logrus.Entry, kubernetescli kubernetes.Interface, arocli aroclient.Interface) *PullSecretReconciler {
return &PullSecretReconciler{
log: log,
kubernetescli: kubernetescli,
arocli: arocli,
}
}

Expand All @@ -56,87 +69,32 @@ func (r *PullSecretReconciler) Reconcile(request ctrl.Request) (ctrl.Result, err
// TODO(mj): controller-runtime master fixes the need for this (https://github.com/kubernetes-sigs/controller-runtime/blob/master/pkg/reconcile/reconcile.go#L93) but it's not yet released.
ctx := context.Background()

mysec, err := r.kubernetescli.CoreV1().Secrets(operator.Namespace).Get(ctx, operator.SecretName, metav1.GetOptions{})
operatorSecret, err := r.kubernetescli.CoreV1().Secrets(operator.Namespace).Get(ctx, operator.SecretName, metav1.GetOptions{})
if err != nil {
return reconcile.Result{}, err
}

return reconcile.Result{}, retry.RetryOnConflict(retry.DefaultRetry, func() error {
ps, isCreate, err := r.pullsecret(ctx)
if err != nil {
return err
}

if ps.Data == nil {
ps.Data = map[string][]byte{}
}

// validate
if !json.Valid(ps.Data[corev1.DockerConfigJsonKey]) {
r.log.Info("pull secret is not valid json - recreating")
delete(ps.Data, corev1.DockerConfigJsonKey)
}

pullsec, changed, err := pullsecret.Merge(string(ps.Data[corev1.DockerConfigJsonKey]), string(mysec.Data[corev1.DockerConfigJsonKey]))
if err != nil {
return err
}

// repair Secret type
if ps.Type != corev1.SecretTypeDockerConfigJson {
ps = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pullSecretName.Name,
Namespace: pullSecretName.Namespace,
},
Type: corev1.SecretTypeDockerConfigJson,
Data: map[string][]byte{},
}
isCreate = true
r.log.Info("pull secret has the wrong type - recreating")
var userSecret *corev1.Secret

// unfortunately the type field is immutable.
err = r.kubernetescli.CoreV1().Secrets(ps.Namespace).Delete(ctx, ps.Name, metav1.DeleteOptions{})
if err != nil {
return err
}

// there is a small risk of crashing here: if that happens, we will
// restart, create a new pull secret, and will have dropped the rest
// of the customer's pull secret on the floor :-(
}
if !isCreate && !changed {
return nil
}

ps.Data[corev1.DockerConfigJsonKey] = []byte(pullsec)
userSecret, err = r.kubernetescli.CoreV1().Secrets(pullSecretName.Namespace).Get(ctx, pullSecretName.Name, metav1.GetOptions{})
if err != nil && !kerrors.IsNotFound(err) {
return reconcile.Result{}, err
}

if isCreate {
r.log.Info("re-creating pull secret")
_, err = r.kubernetescli.CoreV1().Secrets(ps.Namespace).Create(ctx, ps, metav1.CreateOptions{})
} else {
r.log.Info("updating pull secret")
_, err = r.kubernetescli.CoreV1().Secrets(ps.Namespace).Update(ctx, ps, metav1.UpdateOptions{})
}
return err
// reconcile global pull secret
// detects if the global pull secret is broken and fixes it by using backup managed by ARO operator
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
// fix pull secret if its broken to have at least the ARO pull secret
return r.fixAndUpdateGlobalPullSecret(ctx, operatorSecret, userSecret)
})
}

func (r *PullSecretReconciler) pullsecret(ctx context.Context) (*corev1.Secret, bool, error) {
ps, err := r.kubernetescli.CoreV1().Secrets(pullSecretName.Namespace).Get(ctx, pullSecretName.Name, metav1.GetOptions{})
if kerrors.IsNotFound(err) {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pullSecretName.Name,
Namespace: pullSecretName.Namespace,
},
Type: corev1.SecretTypeDockerConfigJson,
}, true, nil
}
if err != nil {
return nil, false, err
return reconcile.Result{}, err
}
return ps, false, nil

redHatKeyCondition := r.buildRedHatKeyCondition(userSecret)
err = controllers.SetCondition(ctx, r.arocli, redHatKeyCondition, operator.RoleMaster)

return reconcile.Result{}, err
}

// SetupWithManager setup our manager
Expand All @@ -162,3 +120,140 @@ func (r *PullSecretReconciler) SetupWithManager(mgr ctrl.Manager) error {
Named(controllers.PullSecretControllerName).
Complete(r)
}

// fixAndUpdateGlobalPullSecret checks the state of the pull secrets, in case of missing or broken ARO pull secret
// it replaces it with working one from controller Secret
func (r *PullSecretReconciler) fixAndUpdateGlobalPullSecret(ctx context.Context, operatorSecret, userSecret *corev1.Secret) (err error) {
if operatorSecret == nil {
return errors.New("nil operator secret, cannot verify userData integrity")
}

var secret *corev1.Secret
create := userSecret == nil
remove := false

// userSecret can happen to have broken type, which means it have to be recreated
// with proper type
if userSecret != nil && (userSecret.Type != corev1.SecretTypeDockerConfigJson || userSecret.Data == nil) {
create = true
remove = true
}

if create {
secret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pullSecretName.Name,
Namespace: pullSecretName.Namespace,
},
Type: corev1.SecretTypeDockerConfigJson,
}
secret.Data = make(map[string][]byte)
} else {
secret = userSecret.DeepCopy()
if !json.Valid(secret.Data[corev1.DockerConfigJsonKey]) {
delete(secret.Data, corev1.DockerConfigJsonKey)
}
}

fixedData, update, err := pullsecret.Merge(string(secret.Data[corev1.DockerConfigJsonKey]), string(operatorSecret.Data[corev1.DockerConfigJsonKey]))
if err != nil {
return err
}

if !create && !update {
return nil
}

if remove {
// unfortunately the type field is immutable.
err := r.kubernetescli.CoreV1().Secrets(secret.Namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{})
if err != nil && !kerrors.IsNotFound(err) {
return err
}
}

secret.Data[corev1.DockerConfigJsonKey] = []byte(fixedData)

if create {
_, err := r.kubernetescli.CoreV1().Secrets(secret.Namespace).Create(ctx, secret, metav1.CreateOptions{})

return err
}

_, err = r.kubernetescli.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{})

return err
}

func (r *PullSecretReconciler) buildRedHatKeyCondition(secret *corev1.Secret) *status.Condition {
// parse keys and validate JSON
parsedKeys, err := pullsecret.UnmarshalSecretData(secret)
foundKeys := []string{}
failed := false
if err != nil {
r.log.Info("pull secret is not valid json - recreating")
failed = true
} else {
foundKeys = r.checkRHRegistryKey(parsedKeys)
}

keyCondition := r.keyCondition(failed, foundKeys)
return keyCondition
}

// checkRHRegistryKey checks whether the rhRegistry key:
// - redhat.registry.io
// - cloud.redhat.com
// is present in the pullSecret
func (r *PullSecretReconciler) checkRHRegistryKey(psData map[string]string) (foundKeys []string) {
if psData == nil {
return foundKeys
}

for _, rhKey := range rhKeys {
for k, v := range psData {
if k == rhKey && len(v) > 0 {
foundKeys = append(foundKeys, rhKey)
}
}

}

return foundKeys
}

func (r *PullSecretReconciler) keyCondition(failed bool, foundKeys []string) *status.Condition {
keyCondition := &status.Condition{
Type: arov1alpha1.RedHatKeyPresent,
Status: corev1.ConditionFalse,
Reason: "CheckFailed",
}

if failed {
keyCondition.Message = "Cannot parse pull-secret"
return keyCondition
}

keyCondition.Reason = "CheckDone"

if len(foundKeys) == 0 {
keyCondition.Message = "No Red Hat key found in pull-secret"
return keyCondition
}

if len(foundKeys) == 0 {
keyCondition.Status = corev1.ConditionFalse
return keyCondition
}

b := strings.Builder{}
for _, key := range foundKeys {
b.WriteString(key)
b.WriteString(",")
}

keyCondition.Status = corev1.ConditionTrue
keyCondition.Message = b.String()

return keyCondition
}
Loading

0 comments on commit 2fac92d

Please sign in to comment.