2.12 nfs

.travis.yml

@@ -2,20 +2,20 @@ sudo: required
 dist: xenial
 language: go
 go:
-  - 1.17.3
+  - 1.17.11
 before_install:
-  - sudo apt-get update -yq
+  - sudo apt-get update -yq || true
   - sudo apt-get install go-md2man -y
   - sudo apt-get install -y awscli
 cache:
   directories:
     - $HOME/.cache/go-build
 script:
   - |
-    if [ "${TRAVIS_BRANCH}" == "master" ]; then
-      export DOCKER_HUB_STORK_TAG=master
-      export DOCKER_HUB_STORK_TEST_TAG=latest
-      export DOCKER_HUB_CMD_EXECUTOR_TAG=master
+    if [ "${TRAVIS_BRANCH}" == "2.12-nfs" ]; then
+      export DOCKER_HUB_STORK_TAG="${TRAVIS_BRANCH}"-dev
+      export DOCKER_HUB_STORK_TEST_TAG="${TRAVIS_BRANCH}"-dev
+      export DOCKER_HUB_CMD_EXECUTOR_TAG="${TRAVIS_BRANCH}"-dev
     else
       export DOCKER_HUB_STORK_TAG=`git rev-parse --short HEAD`
       export DOCKER_HUB_STORK_TEST_TAG=`git rev-parse --short HEAD`

Dockerfile

@@ -18,7 +18,7 @@ RUN python3 -m pip install awscli  && python3 -m pip install rsa --upgrade
 RUN curl -q -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator && \
     chmod +x /usr/local/bin/aws-iam-authenticator
 
-ARG GCLOUD_SDK=google-cloud-sdk-269.0.0-linux-x86_64.tar.gz
+ARG GCLOUD_SDK=google-cloud-sdk-399.0.0-linux-x86_64.tar.gz
 # Remove the test directories
 # Also don't need gsutil
 RUN curl -q -o $GCLOUD_SDK https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/$GCLOUD_SDK && \

Makefile

@@ -29,7 +29,7 @@ ifeq ($(BUILD_TYPE),debug)
 BUILDFLAGS += -gcflags "-N -l"
 endif
 
-RELEASE_VER := 2.7.0
+RELEASE_VER := 2.12.0
 BASE_DIR    := $(shell git rev-parse --show-toplevel)
 GIT_SHA     := $(shell git rev-parse --short HEAD)
 BIN         :=$(BASE_DIR)/bin

cmd/stork/stork.go

@@ -29,6 +29,7 @@ import (
 	"github.com/libopenstorage/stork/pkg/k8sutils"
 	"github.com/libopenstorage/stork/pkg/metrics"
 	"github.com/libopenstorage/stork/pkg/migration"
+	"github.com/libopenstorage/stork/pkg/migration/controllers"
 	"github.com/libopenstorage/stork/pkg/monitor"
 	"github.com/libopenstorage/stork/pkg/objectcontroller"
 	"github.com/libopenstorage/stork/pkg/pvcwatcher"
@@ -41,9 +42,11 @@ import (
 	"github.com/libopenstorage/stork/pkg/webhookadmission"
 	kdmpapi "github.com/portworx/kdmp/pkg/apis/kdmp/v1alpha1"
 	"github.com/portworx/kdmp/pkg/controllers/dataexport"
+	"github.com/portworx/kdmp/pkg/controllers/resourceexport"
 	"github.com/portworx/kdmp/pkg/drivers"
 	"github.com/portworx/kdmp/pkg/jobratelimit"
 	kdmpversion "github.com/portworx/kdmp/pkg/version"
+	"github.com/portworx/sched-ops/k8s/common"
 	schedops "github.com/portworx/sched-ops/k8s/core"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
@@ -143,7 +146,7 @@ func main() {
 			Name:  "application-controller",
 			Usage: "Start the controllers for managing applications (default: true)",
 		},
-		cli.BoolTFlag{
+		cli.BoolFlag{
 			Name:  "px-object-controller",
 			Usage: "Start the px object controller.",
 		},
@@ -184,12 +187,12 @@ func main() {
 		},
 		cli.IntFlag{
 			Name:  "k8s-api-qps",
-			Value: 100,
+			Value: 1000,
 			Usage: "Restrict number of k8s api requests from stork (default: 100 QPS)",
 		},
 		cli.IntFlag{
 			Name:  "k8s-api-burst",
-			Value: 100,
+			Value: 2000,
 			Usage: "Restrict number of k8s api requests from stork (default: 100 Burst)",
 		},
 		cli.BoolTFlag{
@@ -201,6 +204,10 @@ func main() {
 			Value: 4,
 			Usage: "Max threads for apply resources during migration (default: 4)",
 		},
+		cli.BoolTFlag{
+			Name:  controllers.ResourceTransformationControllerName,
+			Usage: "Start the resource transformation controller (default: true)",
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {
@@ -285,6 +292,18 @@ func run(c *cli.Context) {
 	eventBroadcaster.StartRecordingToSink(&core_v1.EventSinkImpl{Interface: k8sClient.CoreV1().Events("")})
 	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, api_v1.EventSource{Component: eventComponentName})
 
+	// Create operator-sdk manager that will manage all controllers.
+	// Setup the controller manager before starting any watches / other controllers
+	mgr, err := manager.New(config, manager.Options{})
+	if err != nil {
+		log.Fatalf("Setup controller manager: %v", err)
+	}
+
+	// Setup scheme for all stork resources
+	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
+		log.Fatalf("Setup scheme failed for stork resources: %v", err)
+	}
+
 	var d volume.Driver
 	if driverName != "" {
 		log.Infof("Using driver %v", driverName)
@@ -333,16 +352,6 @@ func run(c *cli.Context) {
 			}
 		}
 	}
-	// Create operator-sdk manager that will manage all controllers.
-	mgr, err := manager.New(config, manager.Options{})
-	if err != nil {
-		log.Fatalf("Setup controller manager: %v", err)
-	}
-
-	// Setup scheme for all stork resources
-	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
-		log.Fatalf("Setup scheme failed for stork resources: %v", err)
-	}
 
 	runFunc := func(context.Context) {
 		runStork(mgr, d, recorder, c)
@@ -413,6 +422,12 @@ func runStork(mgr manager.Manager, d volume.Driver, recorder record.EventRecorde
 	if err := resourceCollector.Init(nil); err != nil {
 		log.Fatalf("Error initializing ResourceCollector: %v", err)
 	}
+	if err := os.Setenv(common.BurstRate, strconv.Itoa(burst)); err != nil {
+		log.Fatalf("Error setting Burst Rate: %v", err)
+	}
+	if err := os.Setenv(common.QPSRate, strconv.Itoa(qps)); err != nil {
+		log.Fatalf("Error setting Burst Rate: %v", err)
+	}
 	adminNamespace := c.String("admin-namespace")
 	if adminNamespace == "" {
 		adminNamespace = c.String("migration-admin-namespace")
@@ -468,6 +483,13 @@ func runStork(mgr manager.Manager, d volume.Driver, recorder record.EventRecorde
 			}
 		}
 
+		if c.Bool(controllers.ResourceTransformationControllerName) {
+			rt := controllers.NewResourceTransformation(mgr, d, recorder, resourceCollector)
+			if err := rt.Init(mgr); err != nil {
+				log.Fatalf("Error initializing resource transformation controller: %v", err)
+			}
+		}
+
 		if c.Bool("cluster-domain-controllers") {
 			clusterDomains := clusterdomains.ClusterDomains{
 				Driver:   d,
@@ -493,7 +515,7 @@ func runStork(mgr manager.Manager, d volume.Driver, recorder record.EventRecorde
 	if c.Bool("px-object-controller") {
 		objectController := &objectcontroller.ObjectController{}
 		if err := objectController.Init(); err != nil {
-			log.Fatalf("Error initializing px-object-controller : %v", err)
+			log.Warnf("Error initializing px-object-controller : %v", err)
 		}
 	}
 	if c.Bool("kdmp-controller") {
@@ -508,6 +530,14 @@ func runStork(mgr manager.Manager, d volume.Driver, recorder record.EventRecorde
 		if err := dataexport.Init(mgr); err != nil {
 			log.Fatalf("Error initializing kdmp controller: %v", err)
 		}
+		resourceexport, err := resourceexport.NewController(mgr)
+		if err != nil {
+			log.Fatalf("Error initializing resource export controller: %v", err)
+		}
+
+		if err := resourceexport.Init(mgr); err != nil {
+			log.Fatalf("Error initializing resource export controller manager: %v", err)
+		}
 	}
 	ctx := context.Background()
 

drivers/volume/aws/aws.go

@@ -116,11 +116,17 @@ func (a *aws) OwnsPVCForBackup(
 	pvc *v1.PersistentVolumeClaim,
 	cmBackupType string,
 	crBackupType string,
+	blType storkapi.BackupLocationType,
 ) bool {
 	if cmBackupType == storkapi.ApplicationBackupGeneric {
 		// If user has forced the backupType in config map, default to generic always
 		return false
 	}
+	// For AWS volume and backuplocation type is NFS, we will not own.
+	// It will default to kdmp
+	if blType == storkapi.BackupLocationNFS {
+		return false
+	}
 	return a.OwnsPVC(coreOps, pvc)
 }
 
@@ -397,6 +403,7 @@ func (a *aws) DeleteBackup(backup *storkapi.ApplicationBackup) (bool, error) {
 func (a *aws) UpdateMigratedPersistentVolumeSpec(
 	pv *v1.PersistentVolume,
 	vInfo *storkapi.ApplicationRestoreVolumeInfo,
+	namespaceMapping map[string]string,
 ) (*v1.PersistentVolume, error) {
 	if pv.Spec.CSI != nil {
 		pv.Spec.CSI.VolumeHandle = pv.Name

drivers/volume/azure/azure.go

@@ -157,11 +157,17 @@ func (a *azure) OwnsPVCForBackup(
 	pvc *v1.PersistentVolumeClaim,
 	cmBackupType string,
 	crBackupType string,
+	blType storkapi.BackupLocationType,
 ) bool {
 	if cmBackupType == storkapi.ApplicationBackupGeneric {
 		// If user has forced the backupType in config map, default to generic always
 		return false
 	}
+	// For Azure based volume and backuplocation type is NFS, we will not own.
+	// It will default to kdmp
+	if blType == storkapi.BackupLocationNFS {
+		return false
+	}
 	return a.OwnsPVC(coreOps, pvc)
 }
 
@@ -408,6 +414,7 @@ func (a *azure) DeleteBackup(backup *storkapi.ApplicationBackup) (bool, error) {
 func (a *azure) UpdateMigratedPersistentVolumeSpec(
 	pv *v1.PersistentVolume,
 	vInfo *storkapi.ApplicationRestoreVolumeInfo,
+	namespaceMapping map[string]string,
 ) (*v1.PersistentVolume, error) {
 	disk, err := a.diskClient.Get(context.TODO(), a.resourceGroup, pv.Name)
 	if err != nil {

drivers/volume/csi/csi.go

@@ -89,7 +89,7 @@ type BackupObjectv1beta1Csi struct {
 	V1SnapshotRequired     bool
 }
 
-//  GetVolumeSnapshotContent retrieves a backed up volume snapshot
+// GetVolumeSnapshotContent retrieves a backed up volume snapshot
 func (cbo *csiBackupObject) GetVolumeSnapshot(snapshotID string) (interface{}, error) {
 	var vs interface{}
 	var ok bool
@@ -309,7 +309,12 @@ func (c *csi) OwnsPVCForBackup(
 	pvc *v1.PersistentVolumeClaim,
 	cmBackupType string,
 	crBackupType string,
+	blType storkapi.BackupLocationType,
 ) bool {
+	// For CSI volume and backuplocation type is NFS, It will default to kdmp
+	if blType == storkapi.BackupLocationNFS {
+		return false
+	}
 	if cmBackupType == storkapi.ApplicationBackupGeneric || crBackupType == storkapi.ApplicationBackupGeneric {
 		// If user has forced the backupType in config map or applicationbackup CR, default to generic always
 		return false
@@ -1170,6 +1175,7 @@ func (c *csi) DeleteBackup(backup *storkapi.ApplicationBackup) (bool, error) {
 func (c *csi) UpdateMigratedPersistentVolumeSpec(
 	pv *v1.PersistentVolume,
 	vInfo *storkapi.ApplicationRestoreVolumeInfo,
+	namespaceMapping map[string]string,
 ) (*v1.PersistentVolume, error) {
 	return pv, nil
 }

drivers/volume/gcp/gcp.go

@@ -98,11 +98,17 @@ func (g *gcp) OwnsPVCForBackup(
 	pvc *v1.PersistentVolumeClaim,
 	cmBackupType string,
 	crBackupType string,
+	blType storkapi.BackupLocationType,
 ) bool {
 	if cmBackupType == storkapi.ApplicationBackupGeneric {
 		// If user has forced the backupType in config map, default to generic always
 		return false
 	}
+	// For gcp volume and backuplocation type is NFS, we will not own.
+	// It will default to kdmp
+	if blType == storkapi.BackupLocationNFS {
+		return false
+	}
 	return g.OwnsPVC(coreOps, pvc)
 }
 
@@ -347,6 +353,12 @@ func (g *gcp) DeleteBackup(backup *storkapi.ApplicationBackup) (bool, error) {
 		}
 		_, err := service.Snapshots.Delete(vInfo.Options["projectID"], vInfo.BackupID).Do()
 		if err != nil {
+			if gceErr, ok := err.(*googleapi.Error); ok {
+				if gceErr.Code == http.StatusNotFound {
+					// snapshot is already deleted
+					continue
+				}
+			}
 			return true, err
 		}
 	}
@@ -356,6 +368,7 @@ func (g *gcp) DeleteBackup(backup *storkapi.ApplicationBackup) (bool, error) {
 func (g *gcp) UpdateMigratedPersistentVolumeSpec(
 	pv *v1.PersistentVolume,
 	vInfo *storkapi.ApplicationRestoreVolumeInfo,
+	namespaceMapping map[string]string,
 ) (*v1.PersistentVolume, error) {
 	if pv.Spec.CSI != nil {
 		key, err := common.VolumeIDToKey(pv.Spec.CSI.VolumeHandle)