diff --git a/Makefile b/Makefile index 80dbc25..333fa47 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,11 @@ endif CONF_TOOL_VERSION = 4.6 KCONFIG_FILES = $(shell find . -name 'Kconfig') -CLOUD_PROVIDER = $(shell jq -r '.phase1.cloud_provider' .config.json 2>/dev/null) -CLUSTER_NAME = $(shell jq -r '.phase1.cluster_name' .config.json 2>/dev/null) +CONFIG_FILE ?= .config +CONFIG_FILE_ABS := $(realpath $(CONFIG_FILE)) +export CONFIG_JSON_FILE := $(CONFIG_FILE_ABS).json +CLOUD_PROVIDER = $(shell jq -r '.phase1.cloud_provider' ${CONFIG_JSON_FILE} 2>/dev/null) +CLUSTER_NAME = $(shell jq -r '.phase1.cluster_name' ${CONFIG_JSON_FILE} 2>/dev/null) TMP_DIR = $(CLUSTER_NAME)/.tmp default: @@ -32,21 +35,21 @@ config: menuconfig: CONFIG_="." kconfig-mconf Kconfig -.config: $(KCONFIG_FILES) +${CONFIG_FILE_ABS}: $(KCONFIG_FILES) $(MAKE) config -.config.json: .config +${CONFIG_JSON_FILE}: ${CONFIG_FILE_ABS} util/config_to_json $< > $@ -echo-config: .config.json +echo-config: ${CONFIG_JSON_FILE} cat $< -deploy-cluster destroy-cluster: .config.json +deploy-cluster destroy-cluster: ${CONFIG_JSON_FILE} $(MAKE) do WHAT=$@ # For maximum usefulness, use this target with "make -s" to silence any trace output, e.g.: # $ export KUBECONFIG=$(make -s kubeconfig-path) -kubeconfig-path: .config.json +kubeconfig-path: ${CONFIG_JSON_FILE} @$(eval KUBECONFIG_PATH := $(shell pwd)/phase1/$(CLOUD_PROVIDER)/$(CLUSTER_NAME)/kubeconfig.json) @if [ ! -e "$(KUBECONFIG_PATH)" ]; then \ echo "Cannot find kubeconfig file. Have you started a cluster with \"make deploy\" yet?" > /dev/stderr; \ @@ -54,10 +57,10 @@ kubeconfig-path: .config.json fi @echo $(KUBECONFIG_PATH) -validate: .config.json +validate: ${CONFIG_JSON_FILE} KUBECONFIG="$$(pwd)/phase1/$(CLOUD_PROVIDER)/$(CLUSTER_NAME)/kubeconfig.json" ./util/validate -addons: .config.json +addons: ${CONFIG_JSON_FILE} KUBECONFIG="$$(pwd)/phase1/$(CLOUD_PROVIDER)/$(CLUSTER_NAME)/kubeconfig.json" ./phase3/do deploy deploy: | deploy-cluster validate addons @@ -77,7 +80,7 @@ docker-push: docker-build docker push $(IMAGE_NAME):$(IMAGE_VERSION) clean: - ( if [[ -e .config.json ]];then rm -rf phase3/${CLOUD_PROVIDER}/.tmp/ phase1/${CLOUD_PROVIDER}/${CLUSTER_NAME}/; fi ) + ( if [[ -e "${CONFIG_JSON_FILE}" ]];then rm -rf phase3/${CLOUD_PROVIDER}/.tmp/ phase1/${CLOUD_PROVIDER}/${CLUSTER_NAME}/; fi ) fmt: for f in $$(find . -name '*.jsonnet'); do jsonnet fmt -i -n 2 $${f}; done; diff --git a/phase1/Kconfig b/phase1/Kconfig index 5cd3614..2eb46e2 100644 --- a/phase1/Kconfig +++ b/phase1/Kconfig @@ -16,6 +16,12 @@ config phase1.cluster_name This is used for compute instance names as a prefix, and for naming any other cloud provider resources. +config phase1.ssh_user + string "SSH user to login to OS for provisioning" + default "" + help + The SSH user of the target OS. + menuconfig phase1.cloud_provider string "cloud provider: gce, azure or vsphere" default "gce" diff --git a/phase1/gce/all.jsonnet b/phase1/gce/all.jsonnet index af9486a..9e30922 100644 --- a/phase1/gce/all.jsonnet +++ b/phase1/gce/all.jsonnet @@ -1,4 +1,4 @@ -local cfg = import "../../.config.json"; +function(cfg) { ["gce-%(cluster_name)s.tf" % cfg.phase1]: (import "gce.jsonnet")(cfg), } diff --git a/phase1/gce/do b/phase1/gce/do index 70bc52f..1e66f70 100755 --- a/phase1/gce/do +++ b/phase1/gce/do @@ -1,4 +1,4 @@ -#! /bin/bash +#!/bin/bash set -o errexit set -o pipefail @@ -7,7 +7,9 @@ set -x cd "${BASH_SOURCE%/*}" -CLUSTER_NAME=$(jq -r '.phase1.cluster_name' ../../.config.json) +CONFIG_FILE="${CONFIG_JSON_FILE:-../../.config.json}" +CLUSTER_NAME=$(jq -r '.phase1.cluster_name' ${CONFIG_FILE}) +KUBE_CONTEXT_NAME=$(jq -r '.phase2.kube_context_name' ${CONFIG_FILE}) TMP_DIR=${CLUSTER_NAME}/.tmp generate_token() { @@ -17,7 +19,7 @@ generate_token() { gen() { TOKEN=$(generate_token) mkdir -p ${TMP_DIR} - jsonnet -J ../../ --multi ${TMP_DIR} all.jsonnet + jsonnet -J ../../ --multi ${TMP_DIR} --tla-code-file cfg=${CONFIG_FILE} all.jsonnet echo "kubeadm_token = \"$TOKEN\"" > ${CLUSTER_NAME}/terraform.tfvars } @@ -25,7 +27,8 @@ deploy() { gen terraform apply -var-file=${CLUSTER_NAME}/terraform.tfvars -state=${CLUSTER_NAME}/terraform.tfstate ${TMP_DIR} - if [[ "${WAIT_FOR_KUBECONFIG:-}" == "y" ]]; then + PHASE2=$(jq -r '.phase2.provider' ${CONFIG_FILE}) + if [[ "${PHASE2}" == "kubeadm" ]]; then fetch_kubeconfig fi } @@ -36,13 +39,13 @@ deploy() { # terraform. In order to access the cluster remotely, we need to fetch the # proper kubeconfig from the master. fetch_kubeconfig() { - PHASE2=$(jq -r '.phase2.provider' ../../.config.json) - PROJECT=$(jq -r '.phase1.gce.project' ../../.config.json) - ZONE=$(jq -r '.phase1.gce.zone' ../../.config.json) - MASTER=$(jq -r '.phase1.cluster_name' ../../.config.json)-master + PROJECT=$(jq -r '.phase1.gce.project' ${CONFIG_FILE}) + ZONE=$(jq -r '.phase1.gce.zone' ${CONFIG_FILE}) + MASTER=$(jq -r '.phase1.cluster_name' ${CONFIG_FILE})-master + SSHUSER=$(jq -r '.phase1.ssh_user' ${CONFIG_FILE}) - if [[ "$PHASE2" != kubeadm ]]; then - return + if [[ "${SSHUSER}" != "" ]];then + SSHUSER="${SSHUSER}@" fi for tries in {1..60}; do @@ -53,9 +56,18 @@ fetch_kubeconfig() { # succeeds, then stdout is tainted by ssh initialization output, so we echo # a marker to indicate the start of the file, and delete all lines up to and # including it. - if gcloud compute ssh --project "$PROJECT" --zone "$ZONE" "$MASTER" --command "echo STARTFILE; sudo cat /etc/kubernetes/admin.conf" > ${TMP_DIR}/kubeconfig.raw; then + if gcloud compute ssh --project "$PROJECT" --zone "$ZONE" "$SSHUSER$MASTER" --command "echo STARTFILE; sudo cat /etc/kubernetes/admin.conf" > ${TMP_DIR}/kubeconfig.raw; then sed '/^STARTFILE$/,$!d;/^STARTFILE$/d' ${TMP_DIR}/kubeconfig.raw > ${CLUSTER_NAME}/kubeconfig.json echo Successfully fetched kubeconfig. + + if [[ -n "${KUBE_CONTEXT_NAME}" ]]; then + # kubeconfig.json fetched from master node has the user, cluster and context all hardcoded with + # 'kubernetes' keyword. So replace 'kubernetes' with the cluster_name. + # TODO: use `kubectl config rename-(context,cluster,user)` sub-commands whenever they are available + sed -i "s/kubernetes/${CLUSTER_NAME}/g" ${CLUSTER_NAME}/kubeconfig.json + sed -i "s/${CLUSTER_NAME}-admin@${CLUSTER_NAME}/${KUBE_CONTEXT_NAME}/" ${CLUSTER_NAME}/kubeconfig.json + fi + return else sleep 5 diff --git a/phase2/Kconfig b/phase2/Kconfig index 42bf781..b82560e 100644 --- a/phase2/Kconfig +++ b/phase2/Kconfig @@ -39,6 +39,12 @@ config phase2.kubeadm.version Valid options are "stable" or a Google Cloud Storage link to a build. +config phase2.kube_context_name + string "kube context name" + default "" + help + The name of the context in kubeconfig. + endif endmenu diff --git a/phase3/do b/phase3/do index e171930..ea3c910 100755 --- a/phase3/do +++ b/phase3/do @@ -7,11 +7,12 @@ set -x cd "${BASH_SOURCE%/*}" -TMP_DIR=$(jq -r '.phase1.cluster_name' ../.config.json)/.tmp +CONFIG_FILE="${CONFIG_JSON_FILE:-../.config.json}" +TMP_DIR=$(jq -r '.phase1.cluster_name' ${CONFIG_FILE})/.tmp gen() { mkdir -p ${TMP_DIR} - jsonnet --multi ${TMP_DIR} --tla-code-file cfg=../.config.json all.jsonnet + jsonnet --multi ${TMP_DIR} --tla-code-file cfg=${CONFIG_FILE} all.jsonnet } deploy() { diff --git a/util/validate b/util/validate index 15ad1dc..0015831 100755 --- a/util/validate +++ b/util/validate @@ -4,9 +4,11 @@ set -o errexit set -o pipefail set -o nounset +CONFIG_FILE="${CONFIG_JSON_FILE:-.config.json}" + total_time=0 wait_duration=10 -num_nodes="$(( $(cat ./.config.json | jq -r '.phase1.num_nodes') + 1 ))" +num_nodes="$(( $(cat ${CONFIG_FILE} | jq -r '.phase1.num_nodes') + 1 ))" while true; do total_time=$(( ${total_time} + ${wait_duration} )) @@ -15,6 +17,12 @@ while true; do [[ "${hcount}" -ge "${num_nodes}" ]] && echo "Validation: Success!" && exit + # Time out after 20 minutes + if [[ "${total_time}" -ge 1200 ]]; then + echo "Timeout exceeded." + exit 1 + fi + sleep ${wait_duration} done