create a dedicated staging bucket for kops builds

kubernetes · Oct 1, 2023 · eed4487 · eed4487
1 parent d630f9c
commit eed4487
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 5 deletions.
diff --git a/tests/e2e/kubetest2-kops/deployer/build.go b/tests/e2e/kubetest2-kops/deployer/build.go
@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	"k8s.io/klog/v2"
+	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
 	"k8s.io/kops/tests/e2e/pkg/util"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
@@ -37,6 +38,14 @@ func (d *deployer) Build() error {
 	if err := d.init(); err != nil {
 		return err
 	}
+
+	if d.CloudProvider == "gce" && d.createBucket {
+		d.StageLocation = d.stagingStore()
+		if err := gce.EnsureGCSBucket(d.StageLocation, d.GCPProject, true); err != nil {
+			return err
+		}
+	}
+
 	results, err := d.BuildOptions.Build()
 	if err != nil {
 		return err

diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go
@@ -272,14 +272,26 @@ func (d *deployer) stateStore() string {
 			ss = "s3://k8s-kops-prow"
 		case "gce":
 			d.createBucket = true
-			ss = "gs://" + gce.GCSBucketName(d.GCPProject)
+			ss = "gs://" + gce.GCSBucketName(d.GCPProject, "state")
 		case "digitalocean":
 			ss = "do://e2e-kops-space"
 		}
 	}
 	return ss
 }
 
+func (d *deployer) stagingStore() string {
+	sb := os.Getenv("KOPS_STAGING_BUCKET")
+	if sb == "" {
+		switch d.CloudProvider {
+		case "gce":
+			d.createBucket = true
+			sb = "gs://" + gce.GCSBucketName(d.GCPProject, "staging")
+		}
+	}
+	return sb
+}
+
 // the default is $ARTIFACTS if set, otherwise ./_artifacts
 // constructed as an absolute path to help the ginkgo tester because
 // for some reason it needs an absolute path to the kubeconfig

diff --git a/tests/e2e/kubetest2-kops/deployer/down.go b/tests/e2e/kubetest2-kops/deployer/down.go
@@ -56,6 +56,7 @@ func (d *deployer) Down() error {
 
 	if d.CloudProvider == "gce" && d.createBucket {
 		gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
+		gce.DeleteGCSBucket(d.stagingStore(), d.GCPProject)
 	}
 
 	if d.boskos != nil {

diff --git a/tests/e2e/kubetest2-kops/deployer/up.go b/tests/e2e/kubetest2-kops/deployer/up.go
@@ -49,7 +49,7 @@ func (d *deployer) Up() error {
 	}
 
 	if d.CloudProvider == "gce" && d.createBucket {
-		if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
+		if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject, false); err != nil {
 			return err
 		}
 	}

diff --git a/tests/e2e/kubetest2-kops/gce/gcs.go b/tests/e2e/kubetest2-kops/gce/gcs.go
@@ -21,12 +21,13 @@ import (
 	"encoding/hex"
 	"os"
 	"strings"
+	"time"
 
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
 
-func GCSBucketName(projectID string) string {
+func GCSBucketName(projectID, prefix string) string {
 	var s string
 	if jobID := os.Getenv("PROW_JOB_ID"); len(jobID) >= 2 {
 		s = jobID[:2]
@@ -35,11 +36,11 @@ func GCSBucketName(projectID string) string {
 		rand.Read(b)
 		s = hex.EncodeToString(b)
 	}
-	bucket := strings.Join([]string{projectID, "state", s}, "-")
+	bucket := strings.Join([]string{projectID, prefix, s}, "-")
 	return bucket
 }
 
-func EnsureGCSBucket(bucketPath, projectID string) error {
+func EnsureGCSBucket(bucketPath, projectID string, public bool) error {
 	lsArgs := []string{
 		"gsutil", "ls", "-b",
 	}
@@ -75,6 +76,22 @@ func EnsureGCSBucket(bucketPath, projectID string) error {
 	if err != nil {
 		return err
 	}
+
+	if public {
+		iamArgs := []string{
+			"gsutil", "iam", "ch", "allUsers:objectViewer",
+		}
+		iamArgs = append(iamArgs, bucketPath)
+		klog.Info(strings.Join(iamArgs, " "))
+		// GCS APIs are strongly consistent but this should help with flakes
+		time.Sleep(10 * time.Second)
+		cmd = exec.Command(iamArgs[0], iamArgs[1:]...)
+		exec.InheritOutput(cmd)
+		err = cmd.Run()
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }