Zookeeper backup controller

Dockerfile

@@ -28,8 +28,13 @@ COPY controllers/ controllers/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /src/${PROJECT_NAME} \
     -ldflags "-X ${REPO_PATH}/pkg/version.Version=${VERSION} -X ${REPO_PATH}/pkg/version.GitSHA=${GIT_SHA}" main.go
 
-FROM ${DOCKER_REGISTRY:+$DOCKER_REGISTRY/}alpine:${ALPINE_VERSION} AS final
+# Backup script
+COPY build_backup/backup.sh /zookeeper/backup.sh
+RUN chmod +x /zookeeper/backup.sh
+# Install tools for backup
+RUN apk update && apk add findutils tar
 
+FROM ${DOCKER_REGISTRY:+$DOCKER_REGISTRY/}alpine:${ALPINE_VERSION} AS final
 
 ARG PROJECT_NAME=zookeeper-operator
 

PROJECT

@@ -1,13 +1,22 @@
 domain: zookeeper.pravega.io
 layout:
 - go.kubebuilder.io/v3
+plugins:
+  manifests.sdk.operatorframework.io/v2: {}
+  scorecard.sdk.operatorframework.io/v2: {}
+projectName: zookeeper-operator
 repo: github.com/pravega/zookeeper-operator
 resources:
 - group: zookeeper.pravega.io
   kind: ZookeeperCluster
   version: v1beta1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: zookeeper.pravega.io
+  group: zookeeper.pravega.io
+  kind: ZookeeperBackup
+  path: github.com/pravega/zookeeper-operator/api/v1beta1
+  version: v1beta1
 version: "3"
-plugins:
- manifests.sdk.operatorframework.io/v2: {}
- scorecard.sdk.operatorframework.io/v2: {}
-projectName: zookeeper-operator

README.md

@@ -18,6 +18,7 @@ The project is currently alpha. While no breaking API changes are currently plan
     * [Upgrade the Zookeeper Operator](#upgrade-the-operator)
     * [Uninstall the Operator](#uninstall-the-operator)
     * [The AdminServer](#the-adminserver)
+    * [Automatic backup (tech preview)](#automatic-backup-tech-preview)
  * [Development](#development)
     * [Build the Operator Image](#build-the-operator-image)
     * [Direct Access to Cluster](#direct-access-to-the-cluster)
@@ -366,6 +367,40 @@ The list of available commands are
 /commands/zabstate
 ```
 
+### Automatic backup (tech preview)
+In Zookeeper operator was implemented dedicated controller responsible for automatic backups of zookeeper data. Current 
+implementation provides periodic copying of transaction logs and snapshots (on-disk representation of the data) to 
+dedicated persistence volume with specified storage class within kubernetes cluster. Advantages and disadvantages of such
+approach described in [article](https://www.elastic.co/blog/zookeeper-backup-a-treatise).
+Exmaple CR of zookeeper backup:
+```yaml
+apiVersion: "zookeeper.pravega.io/v1beta1"
+kind: "ZookeeperCluster"
+metadata:
+  name: "example-backup"
+spec:
+    zookeeperCluster: "zookeeper-cluster"
+    schedule: "0 0 */1 * *"
+    backupsToKeep: "7"
+    dataStorageClass: "backup-class"
+    image:
+      repository: "pravega/zkbackup"
+      tag: "0.1"
+```
+
+Parameters:
+- *zookeeperCluster* (required) - name of zookeeper cluster to backup.
+- *schedule* (optional) - the schedule in Cron format.
+- *backupsToKeep* (optional) - number of stored backups.
+- *dataStorageClass* (required) - storage class used for persistence volume for backups. Storage class and related provisioner should be configured separately.
+- *image* (optional) - image for backup procedure.
+
+Backup controller takes following responsibilities:
+- Provide cluster health check and cancel backup operation if required.
+- Detect ZK leader pod and prepare/reconfigure CronJob configuration (Backup pod should land on node where leader is elected).
+- Schedule CronJob with a backup script.
+- Provide mechanism to periodic checks to make sure CronJob configuration is updated and valid (for example in case of new leader election)
+
 ## Development
 
 ### Build the operator image

api/v1beta1/deepcopy_test.go

@@ -325,3 +325,107 @@ var _ = Describe("ZookeeperCluster DeepCopy", func() {
 
 	})
 })
+
+var _ = Describe("ZookeeperBackup DeepCopy", func() {
+	Context("with defaults", func() {
+		var (
+			str1, str1a, str1b, str2, str2a, str2b string
+			zkBk, zkBkCopy, zkBkCopy2              *v1beta1.ZookeeperBackup
+		)
+		BeforeEach(func() {
+			zkBk = &v1beta1.ZookeeperBackup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "example",
+					Namespace: "default",
+				},
+			}
+			zkBk.Spec = v1beta1.ZookeeperBackupSpec{
+				ZookeeperCluster: "ZK_Cluster1",
+			}
+
+			zkBk.WithDefaults()
+			zkBkCopy = zkBk.DeepCopy()
+			zkBkCopy2 = zkBk.DeepCopy()
+
+			str1 = zkBk.Spec.ZookeeperCluster
+			str1a = zkBkCopy.Spec.ZookeeperCluster
+			str1b = zkBkCopy2.Spec.ZookeeperCluster
+
+			str2 = "0 12 */1 * *"
+			zkBk.Spec.Schedule = str2
+			zkBkCopy.Spec = *zkBk.Spec.DeepCopy()
+			zkBk.Spec.DeepCopyInto(&zkBkCopy2.Spec)
+			str2a = zkBkCopy.Spec.Schedule
+			str2b = zkBkCopy2.Spec.Schedule
+
+			zkBkCopy.Status = *zkBk.Status.DeepCopy()
+			zkBk.Status.DeepCopyInto(&zkBkCopy.Status)
+
+		})
+		It("value of str1, str1a and str1b should be equal", func() {
+			Ω(str1a).To(Equal(str1))
+			Ω(str1b).To(Equal(str1))
+		})
+		It("value of str2, str2a and str2b should be 0 12 */1 * *", func() {
+			Ω(str2a).To(Equal(str2))
+			Ω(str2b).To(Equal(str2))
+		})
+
+		It("checking of nil DeepCopy for ZookeeperBackup", func() {
+			var zk *v1beta1.ZookeeperBackup
+			zkCopy := zk.DeepCopy()
+			Ω(zkCopy).To(BeNil())
+		})
+
+		It("checking of DeepCopyObject for ZookeeperBackup", func() {
+			zkObj := zkBk.DeepCopyObject()
+			Ω(zkObj.GetObjectKind().GroupVersionKind().Version).To(Equal(""))
+		})
+		It("checking of nil DeepCopyObject for ZookeeperBackup", func() {
+			var zk *v1beta1.ZookeeperBackup
+			zkCopy := zk.DeepCopyObject()
+			Ω(zkCopy).To(BeNil())
+		})
+
+		It("checking of nil DeepCopy for ZookeeperBackupList", func() {
+			var backupList *v1beta1.ZookeeperBackupList
+			backupList2 := backupList.DeepCopy()
+			Ω(backupList2).To(BeNil())
+		})
+
+		It("checking of DeepCopyObject for ZookeeperBackupList", func() {
+			var backupList v1beta1.ZookeeperBackupList
+			backupList.ResourceVersion = "v1beta1"
+			backupList2 := backupList.DeepCopyObject()
+			Ω(backupList2).ShouldNot(BeNil())
+		})
+		It("checking of DeepCopyObject for ZookeeperBackupList with items", func() {
+			var backupList v1beta1.ZookeeperBackupList
+			backupList.ResourceVersion = "v1beta1"
+			backupList.Items = []v1beta1.ZookeeperBackup{
+				{
+					Spec: v1beta1.ZookeeperBackupSpec{},
+				},
+			}
+			backupList2 := backupList.DeepCopyObject()
+			Ω(backupList2).ShouldNot(BeNil())
+		})
+		It("checking of nil DeepCopyObject for ZookeeperBackupList", func() {
+			var backupList *v1beta1.ZookeeperBackupList
+			backupList2 := backupList.DeepCopyObject()
+			Ω(backupList2).To(BeNil())
+		})
+
+		It("checking of nil DeepCopy for ZookeeperBackupSpec", func() {
+			var backupSpec *v1beta1.ZookeeperBackupSpec
+			backupSpec2 := backupSpec.DeepCopy()
+			Ω(backupSpec2).To(BeNil())
+		})
+
+		It("checking of nil DeepCopy for ZookeeperBackupStatus", func() {
+			var backupStatus *v1beta1.ZookeeperBackupStatus
+			backupStatus2 := backupStatus.DeepCopy()
+			Ω(backupStatus2).To(BeNil())
+		})
+	})
+})

api/v1beta1/zookeeperbackup_types.go

@@ -0,0 +1,105 @@
+/**
+ * Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ */
+
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// ZookeeperBackupSpec defines the desired state of ZookeeperBackup
+type ZookeeperBackupSpec struct {
+	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+
+	// Name of the ZookeeperCluster to backup
+	// +kubebuilder:validation:Required
+	ZookeeperCluster string `json:"zookeeperCluster"`
+	// Schedule in Cron format
+	// +kubebuilder:default:="0 0 */1 * *"
+	// +optional
+	Schedule string `json:"schedule,omitempty"`
+	// Number of backups to store
+	// +kubebuilder:default:="7"
+	// +optional
+	BackupsToKeep string `json:"backupsToKeep,omitempty"`
+	// Data Storage Capacity
+	// +kubebuilder:default:="1Gi"
+	// +optional
+	DataCapacity string `json:"dataCapacity,omitempty"`
+	// Data Storage Class name
+	// +kubebuilder:validation:Required
+	DataStorageClass string `json:"dataStorageClass,omitempty"`
+
+	// Image for backup procedure
+	Image ContainerImage `json:"image,omitempty"`
+}
+
+func (s *ZookeeperBackupSpec) withDefaults() (changed bool) {
+	if s.Schedule == "" {
+		s.Schedule = "0 0 */1 * *"
+		changed = true
+	}
+	if s.BackupsToKeep == "" {
+		s.BackupsToKeep = "7"
+		changed = true
+	}
+	if s.DataCapacity == "" {
+		s.DataCapacity = "1Gi"
+		changed = true
+	}
+	if s.Image.Repository == "" {
+		s.Image.Repository = "pravega/zookeeper-operator"
+		changed = true
+	}
+	if s.Image.Tag == "" {
+		s.Image.Tag = "latest"
+		changed = true
+	}
+	return changed
+}
+
+// ZookeeperBackupStatus defines the observed state of ZookeeperBackup
+type ZookeeperBackupStatus struct {
+	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+// ZookeeperBackup is the Schema for the zookeeperbackups API
+type ZookeeperBackup struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ZookeeperBackupSpec   `json:"spec,omitempty"`
+	Status ZookeeperBackupStatus `json:"status,omitempty"`
+}
+
+func (z *ZookeeperBackup) WithDefaults() bool {
+	return z.Spec.withDefaults()
+}
+
+//+kubebuilder:object:root=true
+
+// ZookeeperBackupList contains a list of ZookeeperBackup
+type ZookeeperBackupList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ZookeeperBackup `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&ZookeeperBackup{}, &ZookeeperBackupList{})
+}

api/v1beta1/zookeeperbackup_types_test.go

@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ */
+
+package v1beta1_test
+
+import (
+	"github.com/pravega/zookeeper-operator/api/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ZookeeperBackup Types", func() {
+	var zkBk v1beta1.ZookeeperBackup
+	BeforeEach(func() {
+		zkBk = v1beta1.ZookeeperBackup{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "example",
+			},
+		}
+	})
+
+	Context("#WithDefaults", func() {
+		var changed bool
+		BeforeEach(func() {
+			changed = zkBk.WithDefaults()
+		})
+
+		It("should return as changed", func() {
+			Ω(changed).To(BeTrue())
+		})
+
+		It("should have a default schedule (every day)", func() {
+			Ω(zkBk.Spec.Schedule).To(BeEquivalentTo("0 0 */1 * *"))
+		})
+
+		It("should have a default BackupsToKeep number", func() {
+			Ω(zkBk.Spec.BackupsToKeep).To(BeEquivalentTo("7"))
+		})
+
+		It("should have a default DataCapacity size", func() {
+			Ω(zkBk.Spec.DataCapacity).To(BeEquivalentTo("1Gi"))
+		})
+
+		It("should have a default image for backup", func() {
+			Ω(zkBk.Spec.Image.Repository).To(BeEquivalentTo("pravega/zookeeper-operator"))
+			Ω(zkBk.Spec.Image.Tag).To(BeEquivalentTo("latest"))
+		})
+	})
+})