Merge pull request #38 from kitagry/retry-when-timeout

retry when timeout error occurs

controllers/secret.go

@@ -3,26 +3,30 @@ package controllers
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"maps"
+	"time"
 
 	"github.com/GoogleCloudPlatform/berglas/pkg/berglas"
 	batchv1alpha1 "github.com/kitagry/berglas-secret-controller/api/v1alpha1"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 const (
 	secretAnnotationKey = "kitagry.github.io/berglasSecret"
 	secretVersionKey    = "kitagry.github.io/berglasSecretVersion"
+
+	reconcileRetryCount = 3
 )
 
 func (r *BerglasSecretReconciler) reconcileSecret(ctx context.Context, req ctrl.Request, bs *batchv1alpha1.BerglasSecret) error {
 	var secret v1.Secret
 	err := r.Get(ctx, req.NamespacedName, &secret)
-	if errors.IsNotFound(err) {
+	if k8serrors.IsNotFound(err) {
 		return r.createSecret(ctx, req, bs)
 	} else if err != nil {
 		return err
@@ -82,10 +86,30 @@ func (r *BerglasSecretReconciler) resolveBerglasSchemas(ctx context.Context, dat
 			continue
 		}
 
-		plaintext, err := r.Berglas.Resolve(ctx, ref.String())
+		var plaintext []byte
+		for range reconcileRetryCount {
+			plaintext, err = r.Berglas.Resolve(ctx, ref.String())
+			if err == nil {
+				break
+			}
+
+			// timeout error is retryable
+			var netErr interface{ Timeout() bool }
+			if errors.As(err, &netErr) && netErr.Timeout() {
+				r.Log.Info("timeout error occurred, retrying", "key", key, "value", value)
+				time.Sleep(100 * time.Millisecond)
+				continue
+			}
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// timeout error occurred 3 times
 		if err != nil {
 			return nil, err
 		}
+
 		result[key] = string(plaintext)
 	}
 	return result, nil

controllers/secret_test.go

@@ -2,8 +2,11 @@ package controllers
 
 import (
 	"context"
+	"log"
 	"testing"
 
+	"github.com/go-logr/stdr"
+	"github.com/google/go-cmp/cmp"
 	batchv1alpha1 "github.com/kitagry/berglas-secret-controller/api/v1alpha1"
 	mockcontroller "github.com/kitagry/berglas-secret-controller/controllers/mock"
 	"go.uber.org/mock/gomock"
@@ -142,3 +145,57 @@ func TestBerglasSecretReconciler_isChanged(t *testing.T) {
 		})
 	}
 }
+
+func TestBerglasSecretReconciler_resolveBerglasSchemas(t *testing.T) {
+	tests := map[string]struct {
+		data                    map[string]string
+		createMockBerglasClient func(ctrl *gomock.Controller) *mockcontroller.MockberglasClient
+
+		expected    map[string]string
+		expectedErr error
+	}{
+		"Retry timeout error": {
+			data: map[string]string{
+				"some": "berglas://storage/secret",
+			},
+			createMockBerglasClient: func(ctrl *gomock.Controller) *mockcontroller.MockberglasClient {
+				controller := mockcontroller.NewMockberglasClient(ctrl)
+				first := controller.EXPECT().Resolve(gomock.Any(), "berglas://storage/secret").Return([]byte(""), context.DeadlineExceeded)
+				second := controller.EXPECT().Resolve(gomock.Any(), "berglas://storage/secret").Return([]byte("got"), nil)
+				gomock.InOrder(first, second)
+				return controller
+			},
+			expected: map[string]string{
+				"some": "got",
+			},
+			expectedErr: nil,
+		},
+		"Retry timeout error 3 times": {
+			data: map[string]string{
+				"some": "berglas://storage/secret",
+			},
+			createMockBerglasClient: func(ctrl *gomock.Controller) *mockcontroller.MockberglasClient {
+				controller := mockcontroller.NewMockberglasClient(ctrl)
+				controller.EXPECT().Resolve(gomock.Any(), "berglas://storage/secret").Return([]byte(""), context.DeadlineExceeded).Times(reconcileRetryCount)
+				return controller
+			},
+			expected:    nil,
+			expectedErr: context.DeadlineExceeded,
+		},
+	}
+
+	for n, tt := range tests {
+		t.Run(n, func(t *testing.T) {
+			berglasClient := tt.createMockBerglasClient(gomock.NewController(t))
+			reconciler := &BerglasSecretReconciler{Berglas: berglasClient, Log: stdr.New(log.Default())}
+
+			got, err := reconciler.resolveBerglasSchemas(context.Background(), tt.data)
+			if err != tt.expectedErr {
+				t.Errorf("expected %v, but got %v", tt.expectedErr, err)
+			}
+			if diff := cmp.Diff(tt.expected, got); diff != "" {
+				t.Errorf("resolveBerglasSchemas result diff (-expect, +got)\n%s", diff)
+			}
+		})
+	}
+}