Add step to pass api server env vars to spark jobs

api/batch/resource.go

@@ -16,6 +16,7 @@ package batch
 
 import (
 	"fmt"
+	"os"
 	"strconv"
 	"time"
 
@@ -150,7 +151,7 @@ func (t *BatchJobTemplater) createDriverSpec(job *models.PredictionJob) (v1beta2
 		return v1beta2.DriverSpec{}, fmt.Errorf("invalid driver memory request: %s", job.Config.ResourceRequest.DriverMemoryRequest)
 	}
 
-	envVars, err := addEnvVars(job)
+	envVars, err := t.addEnvVars(job)
 	if err != nil {
 		return v1beta2.DriverSpec{}, err
 	}
@@ -194,7 +195,7 @@ func (t *BatchJobTemplater) createExecutorSpec(job *models.PredictionJob) (v1bet
 		return v1beta2.ExecutorSpec{}, fmt.Errorf("invalid executor memory request: %s", job.Config.ResourceRequest.ExecutorMemoryRequest)
 	}
 
-	envVars, err := addEnvVars(job)
+	envVars, err := t.addEnvVars(job)
 	if err != nil {
 		return v1beta2.ExecutorSpec{}, err
 	}
@@ -263,13 +264,16 @@ func getCoreRequest(cpuRequest resource.Quantity) *int32 {
 	return &core
 }
 
-func addEnvVars(job *models.PredictionJob) ([]corev1.EnvVar, error) {
+func (t *BatchJobTemplater) addEnvVars(job *models.PredictionJob) ([]corev1.EnvVar, error) {
 	envVars := []corev1.EnvVar{
 		{
 			Name:  envServiceAccountPathKey,
 			Value: envServiceAccountPath,
 		},
 	}
+	for _, ev := range t.batchConfig.APIServerEnvVars {
+		envVars = append(envVars, corev1.EnvVar{Name: ev, Value: os.Getenv(ev)})
+	}
 	for _, ev := range job.Config.EnvVars.ToKubernetesEnvVars() {
 		if ev.Name == envServiceAccountPathKey {
 			return []corev1.EnvVar{}, fmt.Errorf("environment variable '%s' cannot be changed", ev.Name)

api/config/batch.go

@@ -23,4 +23,6 @@ type BatchConfig struct {
 	Tolerations []v1.Toleration
 	// Node Selectors for Jobs Specification
 	NodeSelectors map[string]string
+	// APIServerEnvVars specifies the environment variables that are propagated from the API server to the Spark job
+	APIServerEnvVars []string
 }