From 9a4dafd944fae41684c3b971ce483c4bf1f83ecc Mon Sep 17 00:00:00 2001 From: Johannes Aubart Date: Mon, 2 Jul 2018 16:43:49 +0200 Subject: [PATCH 1/6] separated image version from project version Before, a new docker image was built with every release of this project. docker_run.sh used the image tagged with the current project version. However, the Dockerfile is assumed to change significantly less often than other files of this project. To prevent unnecessary creation of identical docker images with just different tags, there is now a IMAGE_VERSION file that is independent from the project version and specifies the version of the docker image that should be used. --- .ci/pipeline_definitions | 25 +++++++++++++++++++++++-- .ci/update_image_version | 31 +++++++++++++++++++++++++++++++ IMAGE_VERSION | 1 + docker_build.sh | 2 +- docker_run.sh | 2 +- 5 files changed, 57 insertions(+), 4 deletions(-) create mode 100755 .ci/update_image_version create mode 100644 IMAGE_VERSION diff --git a/.ci/pipeline_definitions b/.ci/pipeline_definitions index 117beed..82273d5 100644 --- a/.ci/pipeline_definitions +++ b/.ci/pipeline_definitions @@ -1,4 +1,4 @@ -gardener-setup-image: +gardener-setup-release-and-image: template: 'default' base_definition: traits: @@ -15,6 +15,27 @@ gardener-setup-image: image: 'eu.gcr.io/gardener-project/gardener/gardener-setup' repo: trigger: false + steps: + update_image_version: + publish_to: ['source'] + variants: + create-minor-release: + traits: + release: + nextversion: 'bump_minor' + create-patch-release: + traits: + release: + nextversion: 'bump_patch' +gardener-setup-release: + template: 'default' + base_definition: + traits: + version: + preprocess: + 'finalise' + repo: + trigger: false steps: ~ variants: create-minor-release: @@ -24,4 +45,4 @@ gardener-setup-image: create-patch-release: traits: release: - nextversion: 'bump_patch' \ No newline at end of file + nextversion: 'bump_patch' diff --git a/.ci/update_image_version b/.ci/update_image_version new file mode 100755 index 0000000..e80818d --- /dev/null +++ b/.ci/update_image_version @@ -0,0 +1,31 @@ +#!/bin/bash -eu +# +# Copyright (c) 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z ${SOURCE_PATH:-""} ]]; then + export SOURCE_PATH="$(readlink -f $(dirname ${0})/..)" +else + export SOURCE_PATH="$(readlink -f "$SOURCE_PATH")" +fi + +NEW_IMAGE_VERSION=$(<$SOURCE_PATH/VERSION) +echo $NEW_IMAGE_VERSION > $SOURCE_PATH/IMAGE_VERSION + +echo "Image version is now $NEW_IMAGE_VERSION!" + +echo "Preparing commit ..." +git add $SOURCE_PATH/IMAGE_VERSION +git commit -m "updating image version to $NEW_IMAGE_VERSION" +echo $(git log -1) \ No newline at end of file diff --git a/IMAGE_VERSION b/IMAGE_VERSION new file mode 100644 index 0000000..7dff5b8 --- /dev/null +++ b/IMAGE_VERSION @@ -0,0 +1 @@ +0.2.1 \ No newline at end of file diff --git a/docker_build.sh b/docker_build.sh index 25ef148..a9bf3b7 100755 --- a/docker_build.sh +++ b/docker_build.sh @@ -15,7 +15,7 @@ # limitations under the License. # read latest version from file -CURRENT_IMAGE_VERSION=$( Date: Tue, 3 Jul 2018 09:52:49 +0200 Subject: [PATCH 2/6] added modularity to the deploy_gardener.sh and undeploy_gardener.sh scripts Both scripts now take a specific step as optional parameter and will then deploy the components beginning with that step. For example, "deploy_gardener.sh identity" will deploy the identity and dashboard components and skip everything before that (cert, helm-tiller, gardener, seed-config). This works the same way for the undeploy script (although the order of the steps is reversed there). --- deploy_gardener.sh | 58 ++++++++++++++++++++++++++------------------ undeploy_gardener.sh | 52 ++++++++++++++++++++++++++------------- 2 files changed, 70 insertions(+), 40 deletions(-) diff --git a/deploy_gardener.sh b/deploy_gardener.sh index ccbf2d7..4d8183a 100755 --- a/deploy_gardener.sh +++ b/deploy_gardener.sh @@ -23,29 +23,41 @@ echo "Setting up the cluster ..." pushd "$LANDSCAPE_COMPONENTS_HOME" 1> /dev/null -# kubify - not yet automated -#./deploy.sh kubify - -# certificates -./deploy.sh cert - -# helm-tiller -./deploy.sh helm-tiller - -# gardener -./deploy.sh gardener - -# register garden cluster as seed cluster -./deploy.sh seed-config - -# identity -./deploy.sh identity - -# dashboard -./deploy.sh dashboard - -# certmanager - there's an extra script for that -#./deploy.sh certmanager +if [ $# -gt 0 ]; then + arg="$1" +else + arg=cert +fi +case $arg in + (cert) + # certificates + ./deploy.sh cert + ;& + (helm-tiller) + # helm-tiller + ./deploy.sh helm-tiller + ;& + (gardener) + # gardener + ./deploy.sh gardener + ;& + (seed-config) + # register garden cluster as seed cluster + ./deploy.sh seed-config + ;& + (identity) + # identity + ./deploy.sh identity + ;& + (dashboard) + # dashboard + ./deploy.sh dashboard + ;; + (*) + # something else + fail "Unknown argument: $arg" + ;; +esac popd 1> /dev/null diff --git a/undeploy_gardener.sh b/undeploy_gardener.sh index 88c096a..df85311 100755 --- a/undeploy_gardener.sh +++ b/undeploy_gardener.sh @@ -18,23 +18,41 @@ echo "Removing components..." pushd "$LANDSCAPE_COMPONENTS_HOME" 1> /dev/null -# dashboard -./deploy.sh dashboard --uninstall - -# identity -./deploy.sh identity --uninstall - -# seed-config -./deploy.sh seed-config --uninstall - -# gardener -./deploy.sh gardener --uninstall - -# helm-tiller -./deploy.sh helm-tiller --uninstall - -# certificates -./deploy.sh cert --uninstall +if [ $# -gt 0 ]; then + arg="$1" +else + arg=dashboard +fi +case $arg in + (dashboard) + # dashboard + ./deploy.sh dashboard --uninstall + ;& + (identity) + # identity + ./deploy.sh identity --uninstall + ;& + (seed-config) + # register garden cluster as seed cluster + ./deploy.sh seed-config --uninstall + ;& + (gardener) + # gardener + ./deploy.sh gardener --uninstall + ;& + (helm-tiller) + # helm-tiller + ./deploy.sh helm-tiller --uninstall + ;& + (cert) + # certificates + ./deploy.sh cert --uninstall + ;; + (*) + # something else + fail "Unknown argument: $arg" + ;; +esac popd 1> /dev/null From 9a84de34fd51cc44948d439e53c90c62b8ff672d Mon Sep 17 00:00:00 2001 From: Johannes Aubart Date: Tue, 3 Jul 2018 10:05:44 +0200 Subject: [PATCH 3/6] minor improvements - the certmanager deploy script now waits until the secrets have been recreated - the seed-config deploy script now generates files also when uninstalling (when deployed from one machine and uninstalling from another, the second one was missing the generated files, which are used for uninstalling and this caused the uninstall to fail) - added color to some outputs using the "debug" function defined in bin/lib/colors - docker_build.sh and docker_run.sh can now be called from anywhere --- components/certmanager/deploy | 18 ++++++++++++++++-- components/gardener/deploy | 2 +- components/kubify/wait_for_cluster.sh | 10 +++++----- components/seed-config/deploy | 16 ++++++++-------- docker_build.sh | 4 ++++ docker_run.sh | 6 +++++- init.sh | 1 + 7 files changed, 40 insertions(+), 17 deletions(-) diff --git a/components/certmanager/deploy b/components/certmanager/deploy index 61c2028..5bfeee0 100755 --- a/components/certmanager/deploy +++ b/components/certmanager/deploy @@ -59,8 +59,6 @@ if [ $# -gt 1 ] && [ $2 == "-u" -o $2 == "--uninstall" ]; then # uninstall certm $LANDSCAPE_COMPONENTS_HOME/deploy.sh dashboard # delete certmanager stuff - kubectl delete clusterissuer --all - kubectl -n certmanager delete secret --all kubectl delete ns certmanager else # render issuer file @@ -103,4 +101,20 @@ else else echo "INFO: Line '--oidc-ca-file=/etc/kubernetes/secrets/ca.crt' not found in daemonset kube-apiserver." fi + + # wait until secrets have been recreated + max_retry_time=300 + retry_stop=$(($(date +%s) + max_retry_time)) + success=false + while [[ $(date +%s) -lt $retry_stop ]]; do + sleep 10 + if $(kubectl -n garden get secret gardener-dashboard-tls &> /dev/null) && $(kubectl -n kube-system get secret identity-tls &> /dev/null); then + success=true + break; + fi + debug "Secrets gardener-dashboard-tls and identity-tls not recreated yet. Waiting ..." + done + if ! $success; then + fail "Secrets gardener-dashboard-tls and identity-tls not created within $max_retry_time seconds!" + fi fi \ No newline at end of file diff --git a/components/gardener/deploy b/components/gardener/deploy index ca76aa2..d7c40bd 100755 --- a/components/gardener/deploy +++ b/components/gardener/deploy @@ -66,7 +66,7 @@ else # install gardener success=true break; fi - echo -e "$(debug Gardener API server not yet reachable. Waiting...)" + debug "Gardener API server not yet reachable. Waiting..." done if ! $success; then fail "Gardener API server did not become reachable within $max_retry_time seconds!" diff --git a/components/kubify/wait_for_cluster.sh b/components/kubify/wait_for_cluster.sh index 73280e7..2e75271 100755 --- a/components/kubify/wait_for_cluster.sh +++ b/components/kubify/wait_for_cluster.sh @@ -29,7 +29,7 @@ while [[ $(date +%s) -lt $retry_stop ]]; do success=true break; fi - echo "Cluster not yet reachable. Waiting ..." + debug "Cluster not yet reachable. Waiting ..." sleep 30 done if ! $success; then @@ -53,7 +53,7 @@ while [[ $(date +%s) -lt $retry_stop ]]; do ((phase=$phase-1)) || true continue; fi - echo "Amount of api server pods ($api_count) doesn't equal specified amount of master nodes ($master_count) yet. Waiting ..." + debug "Amount of api server pods ($api_count) doesn't equal specified amount of master nodes ($master_count) yet. Waiting ..." ;; (2) # check: #etcd == #master nodes etcd_count=$(kubectl -n kube-system get pods | grep -i -E "kube-etcd-.... " | wc -l) &> /dev/null @@ -62,7 +62,7 @@ while [[ $(date +%s) -lt $retry_stop ]]; do ((phase=$phase-1)) || true continue; fi - echo "Amount of etcd pods ($etcd_count) doesn't equal specified amount of master nodes ($master_count) yet. Waiting ..." + debug "Amount of etcd pods ($etcd_count) doesn't equal specified amount of master nodes ($master_count) yet. Waiting ..." ;; (1) # check: #ingress == #worker nodes ingress_count=$(kubectl -n nginx-ingress get pods | grep -i nginx-ingress-controller | wc -l) &> /dev/null @@ -71,7 +71,7 @@ while [[ $(date +%s) -lt $retry_stop ]]; do ((phase=$phase-1)) || true continue; fi - echo "Amount of ingress pods ($ingress_count) doesn't equal specified amount of worker nodes ($worker_count) yet. Waiting ..." + debug "Amount of ingress pods ($ingress_count) doesn't equal specified amount of worker nodes ($worker_count) yet. Waiting ..." ;; (0) # check: #pods == #running pods pod_count=$(kubectl get pods --all-namespaces | wc -l) &> /dev/null @@ -83,7 +83,7 @@ while [[ $(date +%s) -lt $retry_stop ]]; do success=true break; fi - echo "$running_pod_count of $pod_count pods are running. Waiting ..." + debug "$running_pod_count of $pod_count pods are running. Waiting ..." ;; (*) # just decrease phase echo "No valid phase: $phase" diff --git a/components/seed-config/deploy b/components/seed-config/deploy index 55d461a..16047c2 100755 --- a/components/seed-config/deploy +++ b/components/seed-config/deploy @@ -14,20 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +for i in ${COMPONENT_TEMPLATE_HOME}/${CLOUD_VARIANT}/*.tmpl ; do + filet=$(basename $i) + file=${filet::-5} + if [ ! -f ${COMPONENT_STATE_HOME}/$file ]; then # create yaml if it doesn't exist + mako-render $i > ${COMPONENT_STATE_HOME}/$file + fi +done + if [ $# -gt 1 ] && [ $2 == "-u" -o $2 == "--uninstall" ]; then # uninstall seed-config for i in $(ls -r ${COMPONENT_STATE_HOME}/*.yaml); do # iterate over files in reverse order kubectl delete -f $i --ignore-not-found done rm -rf $COMPONENT_STATE_HOME else # install seed-config - for i in ${COMPONENT_TEMPLATE_HOME}/${CLOUD_VARIANT}/*.tmpl ; do - filet=$(basename $i) - file=${filet::-5} - mako-render $i > ${COMPONENT_STATE_HOME}/$file - done - - # deploy - for i in ${COMPONENT_STATE_HOME}/*.yaml ; do echo deploying $i kubectl apply -f $i diff --git a/docker_build.sh b/docker_build.sh index a9bf3b7..645ae21 100755 --- a/docker_build.sh +++ b/docker_build.sh @@ -14,8 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +pushd $(dirname ${0}) 1> /dev/null + # read latest version from file CURRENT_IMAGE_VERSION=$( /dev/null \ No newline at end of file diff --git a/docker_run.sh b/docker_run.sh index 3df3acd..9152eb8 100755 --- a/docker_run.sh +++ b/docker_run.sh @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +pushd $(dirname ${0}) 1> /dev/null + # read latest version from file CURRENT_IMAGE_VERSION=$( /dev/null \ No newline at end of file diff --git a/init.sh b/init.sh index e95af73..6111ff7 100755 --- a/init.sh +++ b/init.sh @@ -68,5 +68,6 @@ source ${SETUP_REPO_PATH}/bin/common export -f fail export -f error export -f color +export -f debug echo "Environment variables configured!" \ No newline at end of file From ee76145e771bff2f4c3fb027dbbf3669e83737c0 Mon Sep 17 00:00:00 2001 From: Johannes Aubart Date: Tue, 3 Jul 2018 12:03:23 +0200 Subject: [PATCH 4/6] added information on letsencrypt rate limits to README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 615756a..4389b0c 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,13 @@ letsencrypt certificates for both the identity and dashboard ingresses: After one to two minutes valid certificates should be installed. +Letsencrypt [limits](https://letsencrypt.org/docs/rate-limits/) how many certificates you can get +for the same host within a short time. To avoid hitting these limits, you can use the letsencrypt +staging server for testing, which has a significantly higher rate limit but produces untrusted +certificates. +To use the staging server, change the URL in `components/certmanager/cert-manager-issuer.yaml.tmpl` +to `https://acme-staging-v02.api.letsencrypt.org/directory`. + # Tearing Down the Landscape Make sure that you delete all shoot clusters prior to tearing down the From 8698deefdf3349993110d2308eea83c35fc153af Mon Sep 17 00:00:00 2001 From: Johannes Aubart Date: Thu, 5 Jul 2018 09:01:46 +0200 Subject: [PATCH 5/6] added deletion of state folder to certmanager undeploy script --- components/certmanager/deploy | 1 + 1 file changed, 1 insertion(+) diff --git a/components/certmanager/deploy b/components/certmanager/deploy index 5bfeee0..822c7ae 100755 --- a/components/certmanager/deploy +++ b/components/certmanager/deploy @@ -60,6 +60,7 @@ if [ $# -gt 1 ] && [ $2 == "-u" -o $2 == "--uninstall" ]; then # uninstall certm # delete certmanager stuff kubectl delete ns certmanager + rm -rf $COMPONENT_STATE_HOME else # render issuer file mako-render ${COMPONENT_TEMPLATE_HOME}/cert-manager-issuer.yaml.tmpl >${COMPONENT_STATE_HOME}/cert-manager-issuer.yaml From 38d67414eee441b1acba6739aa105c95380cc075 Mon Sep 17 00:00:00 2001 From: Johannes Aubart Date: Thu, 5 Jul 2018 14:28:14 +0200 Subject: [PATCH 6/6] added automation for sessionAffinity fix At the end of the deploy_kubify.sh script, the kubernetes service will automatically be patched to apply the workaround for the sessionAffinity problem. --- README.md | 19 ++++++------------- deploy_kubify.sh | 11 ++++++++--- session_affinity_fix.sh | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 16 deletions(-) create mode 100755 session_affinity_fix.sh diff --git a/README.md b/README.md index 4389b0c..3c2b0bc 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,6 @@ k8s/bin/tf destroy -force setup/cleanup.sh ``` -Don't forget the [workaround](#workaround)! - - ## Step 1: Clone the Repositories and get Dependencies Get the `landscape-setup-template` from GitHub and initialize the @@ -127,19 +124,13 @@ kube-system kube-apiserver-hcdnc [...] ``` -## Step 4.5: Workaround +## Step 4.5: Workaround (Automated) There is currently an issue with session affinities in Kubernetes, which can break your cluster. While the problem has been fixed (see https://github.com/kubernetes/kubernetes/commit/f2405cf2942739996af2bb76347c2cb0641153aa), -this commit is not yet included in a release. +the corresponding Kubernetes version is not yet included in this project. -Until that happens, use the following workaround: - -``` -kubectl edit svc kubernetes -``` - -Delete the following lines: +Until that happens, the workaround is to remove the following lines from the `kubernetes` service: ``` sessionAffinity: ClientIP @@ -148,7 +139,9 @@ Delete the following lines: timeoutSeconds: 10800 ``` -Kubernetes will automatically add `sessionAffinity: None`. +Kubernetes will add `sessionAffinity: None` on itself. + +This will happen automatically at the end of the `deploy_kubify.sh` script. ## Step 5-9: Gardener Setup (Automated) diff --git a/deploy_kubify.sh b/deploy_kubify.sh index a692f6e..fff35fb 100755 --- a/deploy_kubify.sh +++ b/deploy_kubify.sh @@ -14,8 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -cd components +pushd "$LANDSCAPE_COMPONENTS_HOME" 1> /dev/null + ./deploy.sh kubify -cd .. -echo "Cluster successfully set up!" \ No newline at end of file +echo "Cluster successfully set up!" + +echo "" +$SETUP_REPO_PATH/session_affinity_fix.sh + +popd 1> /dev/null \ No newline at end of file diff --git a/session_affinity_fix.sh b/session_affinity_fix.sh new file mode 100755 index 0000000..a6f4601 --- /dev/null +++ b/session_affinity_fix.sh @@ -0,0 +1,19 @@ +#!/bin/bash -eu +# +# Copyright (c) 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Removing sessionAffinity from kubernetes service ..." +kubectl patch svc kubernetes --type json -p='[{"op": "remove", "path": "/spec/sessionAffinity"}]' +kubectl get svc kubernetes -o yaml \ No newline at end of file