Merge pull request kubernetes-retired#429 from shashidharatd/gce-fede…

…ration

Deploy multiple clusters in parallel and few other fixes

Makefile

@@ -19,8 +19,11 @@ endif
 CONF_TOOL_VERSION = 4.6
 KCONFIG_FILES = $(shell find . -name 'Kconfig')
 
-CLOUD_PROVIDER = $(shell jq -r '.phase1.cloud_provider' .config.json 2>/dev/null)
-CLUSTER_NAME = $(shell jq -r '.phase1.cluster_name' .config.json 2>/dev/null)
+CONFIG_FILE ?= .config
+CONFIG_FILE_ABS := $(realpath $(CONFIG_FILE))
+export CONFIG_JSON_FILE := $(CONFIG_FILE_ABS).json
+CLOUD_PROVIDER = $(shell jq -r '.phase1.cloud_provider' ${CONFIG_JSON_FILE} 2>/dev/null)
+CLUSTER_NAME = $(shell jq -r '.phase1.cluster_name' ${CONFIG_JSON_FILE} 2>/dev/null)
 TMP_DIR = $(CLUSTER_NAME)/.tmp
 
 default:
@@ -32,32 +35,32 @@ config:
 menuconfig:
 	CONFIG_="." kconfig-mconf Kconfig
 
-.config: $(KCONFIG_FILES)
+${CONFIG_FILE_ABS}: $(KCONFIG_FILES)
 	$(MAKE) config
 
-.config.json: .config
+${CONFIG_JSON_FILE}: ${CONFIG_FILE_ABS}
 	util/config_to_json $< > $@
 
-echo-config: .config.json
+echo-config: ${CONFIG_JSON_FILE}
 	cat $<
 
-deploy-cluster destroy-cluster: .config.json
+deploy-cluster destroy-cluster: ${CONFIG_JSON_FILE}
 	$(MAKE) do WHAT=$@
 
 # For maximum usefulness, use this target with "make -s" to silence any trace output, e.g.:
 #   $ export KUBECONFIG=$(make -s kubeconfig-path)
-kubeconfig-path: .config.json
+kubeconfig-path: ${CONFIG_JSON_FILE}
 	@$(eval KUBECONFIG_PATH := $(shell pwd)/phase1/$(CLOUD_PROVIDER)/$(CLUSTER_NAME)/kubeconfig.json)
 	@if [ ! -e "$(KUBECONFIG_PATH)" ]; then \
 		echo "Cannot find kubeconfig file. Have you started a cluster with \"make deploy\" yet?" > /dev/stderr; \
 		exit 1; \
 	fi
 	@echo $(KUBECONFIG_PATH)
 
-validate: .config.json
+validate: ${CONFIG_JSON_FILE}
 	KUBECONFIG="$$(pwd)/phase1/$(CLOUD_PROVIDER)/$(CLUSTER_NAME)/kubeconfig.json" ./util/validate
 
-addons: .config.json
+addons: ${CONFIG_JSON_FILE}
 	KUBECONFIG="$$(pwd)/phase1/$(CLOUD_PROVIDER)/$(CLUSTER_NAME)/kubeconfig.json" ./phase3/do deploy
 
 deploy: | deploy-cluster validate addons
@@ -77,7 +80,7 @@ docker-push: docker-build
 	docker push $(IMAGE_NAME):$(IMAGE_VERSION)
 
 clean:
-	( if [[ -e .config.json ]];then rm -rf phase3/${CLOUD_PROVIDER}/.tmp/ phase1/${CLOUD_PROVIDER}/${CLUSTER_NAME}/; fi )
+	( if [[ -e "${CONFIG_JSON_FILE}" ]];then rm -rf phase3/${CLOUD_PROVIDER}/.tmp/ phase1/${CLOUD_PROVIDER}/${CLUSTER_NAME}/; fi )
 
 fmt:
 	for f in $$(find . -name '*.jsonnet'); do jsonnet fmt -i -n 2 $${f}; done;

phase1/Kconfig

@@ -16,6 +16,12 @@ config phase1.cluster_name
 	  This is used for compute instance names as a prefix,
 	  and for naming any other cloud provider resources.
 
+config phase1.ssh_user
+	string "SSH user to login to OS for provisioning"
+	default ""
+	help
+	  The SSH user of the target OS.
+
 menuconfig phase1.cloud_provider
 	string "cloud provider: gce, azure or vsphere"
 	default "gce"

phase1/gce/all.jsonnet

@@ -1,4 +1,4 @@
-local cfg = import "../../.config.json";
+function(cfg)
 {
   ["gce-%(cluster_name)s.tf" % cfg.phase1]: (import "gce.jsonnet")(cfg),
 }

phase1/gce/do

@@ -1,4 +1,4 @@
-#! /bin/bash
+#!/bin/bash
 
 set -o errexit
 set -o pipefail
@@ -7,7 +7,9 @@ set -x
 
 cd "${BASH_SOURCE%/*}"
 
-CLUSTER_NAME=$(jq -r '.phase1.cluster_name' ../../.config.json)
+CONFIG_FILE="${CONFIG_JSON_FILE:-../../.config.json}"
+CLUSTER_NAME=$(jq -r '.phase1.cluster_name' ${CONFIG_FILE})
+KUBE_CONTEXT_NAME=$(jq -r '.phase2.kube_context_name' ${CONFIG_FILE})
 TMP_DIR=${CLUSTER_NAME}/.tmp
 
 generate_token() {
@@ -17,15 +19,16 @@ generate_token() {
 gen() {
   TOKEN=$(generate_token)
   mkdir -p ${TMP_DIR}
-  jsonnet -J ../../ --multi ${TMP_DIR} all.jsonnet
+  jsonnet -J ../../ --multi ${TMP_DIR} --tla-code-file cfg=${CONFIG_FILE} all.jsonnet
   echo "kubeadm_token = \"$TOKEN\"" > ${CLUSTER_NAME}/terraform.tfvars
 }
 
 deploy() {
   gen
   terraform apply -var-file=${CLUSTER_NAME}/terraform.tfvars -state=${CLUSTER_NAME}/terraform.tfstate ${TMP_DIR}
 
-  if [[ "${WAIT_FOR_KUBECONFIG:-}" == "y" ]]; then
+  PHASE2=$(jq -r '.phase2.provider' ${CONFIG_FILE})
+  if [[ "${PHASE2}" == "kubeadm" ]]; then
     fetch_kubeconfig
   fi
 }
@@ -36,13 +39,13 @@ deploy() {
 # terraform. In order to access the cluster remotely, we need to fetch the
 # proper kubeconfig from the master.
 fetch_kubeconfig() {
-  PHASE2=$(jq -r '.phase2.provider' ../../.config.json)
-  PROJECT=$(jq -r '.phase1.gce.project' ../../.config.json)
-  ZONE=$(jq -r '.phase1.gce.zone' ../../.config.json)
-  MASTER=$(jq -r '.phase1.cluster_name' ../../.config.json)-master
+  PROJECT=$(jq -r '.phase1.gce.project' ${CONFIG_FILE})
+  ZONE=$(jq -r '.phase1.gce.zone' ${CONFIG_FILE})
+  MASTER=$(jq -r '.phase1.cluster_name' ${CONFIG_FILE})-master
+  SSHUSER=$(jq -r '.phase1.ssh_user' ${CONFIG_FILE})
 
-  if [[ "$PHASE2" != kubeadm ]]; then
-    return
+  if [[ "${SSHUSER}" != "" ]];then
+    SSHUSER="${SSHUSER}@"
   fi
 
   for tries in {1..60}; do
@@ -53,9 +56,18 @@ fetch_kubeconfig() {
     # succeeds, then stdout is tainted by ssh initialization output, so we echo
     # a marker to indicate the start of the file, and delete all lines up to and
     # including it.
-    if gcloud compute ssh --project "$PROJECT" --zone "$ZONE" "$MASTER" --command "echo STARTFILE; sudo cat /etc/kubernetes/admin.conf" > ${TMP_DIR}/kubeconfig.raw; then
+    if gcloud compute ssh --project "$PROJECT" --zone "$ZONE" "$SSHUSER$MASTER" --command "echo STARTFILE; sudo cat /etc/kubernetes/admin.conf" > ${TMP_DIR}/kubeconfig.raw; then
       sed '/^STARTFILE$/,$!d;/^STARTFILE$/d' ${TMP_DIR}/kubeconfig.raw > ${CLUSTER_NAME}/kubeconfig.json
       echo Successfully fetched kubeconfig.
+
+      if [[ -n "${KUBE_CONTEXT_NAME}" ]]; then
+        # kubeconfig.json fetched from master node has the user, cluster and context all hardcoded with
+        # 'kubernetes' keyword. So replace 'kubernetes' with the cluster_name.
+        # TODO: use `kubectl config rename-(context,cluster,user)` sub-commands whenever they are available
+        sed -i "s/kubernetes/${CLUSTER_NAME}/g" ${CLUSTER_NAME}/kubeconfig.json
+        sed -i "s/${CLUSTER_NAME}-admin@${CLUSTER_NAME}/${KUBE_CONTEXT_NAME}/" ${CLUSTER_NAME}/kubeconfig.json
+      fi
+
       return
     else
       sleep 5

phase2/Kconfig

@@ -39,6 +39,12 @@ config phase2.kubeadm.version
 
 	  Valid options are "stable" or a Google Cloud Storage link to a build.
 
+config phase2.kube_context_name
+    string "kube context name"
+    default ""
+    help
+	  The name of the context in kubeconfig.
+
 endif
 
 endmenu

phase3/do

@@ -7,11 +7,12 @@ set -x
 
 cd "${BASH_SOURCE%/*}"
 
-TMP_DIR=$(jq -r '.phase1.cluster_name' ../.config.json)/.tmp
+CONFIG_FILE="${CONFIG_JSON_FILE:-../.config.json}"
+TMP_DIR=$(jq -r '.phase1.cluster_name' ${CONFIG_FILE})/.tmp
 
 gen() {
 	mkdir -p ${TMP_DIR}
-	jsonnet --multi ${TMP_DIR} --tla-code-file cfg=../.config.json all.jsonnet
+	jsonnet --multi ${TMP_DIR} --tla-code-file cfg=${CONFIG_FILE} all.jsonnet
 }
 
 deploy() {

util/validate

@@ -4,9 +4,11 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+CONFIG_FILE="${CONFIG_JSON_FILE:-.config.json}"
+
 total_time=0
 wait_duration=10
-num_nodes="$(( $(cat ./.config.json | jq -r '.phase1.num_nodes') + 1 ))"
+num_nodes="$(( $(cat ${CONFIG_FILE} | jq -r '.phase1.num_nodes') + 1 ))"
 
 while true; do
   total_time=$(( ${total_time} + ${wait_duration} ))
@@ -15,6 +17,12 @@ while true; do
 
   [[ "${hcount}" -ge "${num_nodes}" ]] && echo "Validation: Success!" && exit
 
+  # Time out after 20 minutes
+  if [[ "${total_time}" -ge 1200 ]]; then
+    echo "Timeout exceeded."
+    exit 1
+  fi
+
   sleep ${wait_duration}
 done