diff --git a/controllers/secret.go b/controllers/secret.go index f7bac1a..79374dc 100644 --- a/controllers/secret.go +++ b/controllers/secret.go @@ -3,13 +3,15 @@ package controllers import ( "context" "encoding/json" + "errors" "fmt" "maps" + "time" "github.com/GoogleCloudPlatform/berglas/pkg/berglas" batchv1alpha1 "github.com/kitagry/berglas-secret-controller/api/v1alpha1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" ) @@ -17,12 +19,14 @@ import ( const ( secretAnnotationKey = "kitagry.github.io/berglasSecret" secretVersionKey = "kitagry.github.io/berglasSecretVersion" + + reconcileRetryCount = 3 ) func (r *BerglasSecretReconciler) reconcileSecret(ctx context.Context, req ctrl.Request, bs *batchv1alpha1.BerglasSecret) error { var secret v1.Secret err := r.Get(ctx, req.NamespacedName, &secret) - if errors.IsNotFound(err) { + if k8serrors.IsNotFound(err) { return r.createSecret(ctx, req, bs) } else if err != nil { return err @@ -82,10 +86,30 @@ func (r *BerglasSecretReconciler) resolveBerglasSchemas(ctx context.Context, dat continue } - plaintext, err := r.Berglas.Resolve(ctx, ref.String()) + var plaintext []byte + for range reconcileRetryCount { + plaintext, err = r.Berglas.Resolve(ctx, ref.String()) + if err == nil { + break + } + + // timeout error is retryable + var netErr interface{ Timeout() bool } + if errors.As(err, &netErr) && netErr.Timeout() { + r.Log.Info("timeout error occurred, retrying", "key", key, "value", value) + time.Sleep(100 * time.Millisecond) + continue + } + if err != nil { + return nil, err + } + } + + // timeout error occurred 3 times if err != nil { return nil, err } + result[key] = string(plaintext) } return result, nil diff --git a/controllers/secret_test.go b/controllers/secret_test.go index f96ff2b..0263cb8 100644 --- a/controllers/secret_test.go +++ b/controllers/secret_test.go @@ -2,8 +2,11 @@ package controllers import ( "context" + "log" "testing" + "github.com/go-logr/stdr" + "github.com/google/go-cmp/cmp" batchv1alpha1 "github.com/kitagry/berglas-secret-controller/api/v1alpha1" mockcontroller "github.com/kitagry/berglas-secret-controller/controllers/mock" "go.uber.org/mock/gomock" @@ -142,3 +145,57 @@ func TestBerglasSecretReconciler_isChanged(t *testing.T) { }) } } + +func TestBerglasSecretReconciler_resolveBerglasSchemas(t *testing.T) { + tests := map[string]struct { + data map[string]string + createMockBerglasClient func(ctrl *gomock.Controller) *mockcontroller.MockberglasClient + + expected map[string]string + expectedErr error + }{ + "Retry timeout error": { + data: map[string]string{ + "some": "berglas://storage/secret", + }, + createMockBerglasClient: func(ctrl *gomock.Controller) *mockcontroller.MockberglasClient { + controller := mockcontroller.NewMockberglasClient(ctrl) + first := controller.EXPECT().Resolve(gomock.Any(), "berglas://storage/secret").Return([]byte(""), context.DeadlineExceeded) + second := controller.EXPECT().Resolve(gomock.Any(), "berglas://storage/secret").Return([]byte("got"), nil) + gomock.InOrder(first, second) + return controller + }, + expected: map[string]string{ + "some": "got", + }, + expectedErr: nil, + }, + "Retry timeout error 3 times": { + data: map[string]string{ + "some": "berglas://storage/secret", + }, + createMockBerglasClient: func(ctrl *gomock.Controller) *mockcontroller.MockberglasClient { + controller := mockcontroller.NewMockberglasClient(ctrl) + controller.EXPECT().Resolve(gomock.Any(), "berglas://storage/secret").Return([]byte(""), context.DeadlineExceeded).Times(reconcileRetryCount) + return controller + }, + expected: nil, + expectedErr: context.DeadlineExceeded, + }, + } + + for n, tt := range tests { + t.Run(n, func(t *testing.T) { + berglasClient := tt.createMockBerglasClient(gomock.NewController(t)) + reconciler := &BerglasSecretReconciler{Berglas: berglasClient, Log: stdr.New(log.Default())} + + got, err := reconciler.resolveBerglasSchemas(context.Background(), tt.data) + if err != tt.expectedErr { + t.Errorf("expected %v, but got %v", tt.expectedErr, err) + } + if diff := cmp.Diff(tt.expected, got); diff != "" { + t.Errorf("resolveBerglasSchemas result diff (-expect, +got)\n%s", diff) + } + }) + } +}