Automatic recovery from permanent failures of etcd3 nodes (kubernetes…

…-retired#417)

## Features

* Automatic recovery from any number of permanently failed etcd nodes (when `etcd.disasterRecovery.automated` is set to true
  * Currently, it is considered to be a "permanent failure" when health checks to an etcd member is failing longer than the threshold(10 seconds by default)

## Notable changes

* Upgraded etcd to v3
* kube-apiserver uses etcd3 API rather than the former etcd2 API for writing v3 keys when etcd3 is chosen for the etcd cluster
  * Although you can stick with v2 api + v3 etcd(then kube-apiserver writes v2 keys in etcd v3!), the current implementation of etcd snapshot/restore doesn't support etcd v2 data. That's why I made this change
  * etcd3 will be the default storage backend since k8s 1.6 anyways
* Wrote a lengthy `etcdadm` script to automate etcd member health checking, saving snapshots, recovering from up to `N/2` permanently failed nodes(=replace failed members one by one), recovering from more than `N/2` permanently failed nodes(=bootstrapping a brand-new cluster with the latest snapshot)

## Changelog

* Add etcdadm for scripting various etcd3 administration tasks to achieve high availability

* Enable etcd3 paired with the `etcdadm reconfigure` service

* Periodically run etcdadm (save|check) when `etcd.snapshot.automated` or `etcd.disasterRecovery.automated` is set to true, respectively

* Fail-fast etcd-member when `etcdadm reconfigure` fails

* When etcd3 is chosen, use etcd v3 API for communication between etcd and apiserver
k8s state is persisted in etcd3 data rather than etcd2 data in an etcd3 cluster(etcd3 can serve both v2 api and v3 api for accessing v2 data and v3 data respectively)

* Turn off automatic Container Linux updates on Etcd nodes

* Support both v2 and v3 of etcd & Allow switching etcd version (for now)

* Fix a validation for awsNodeLabels

* e2e: Add a convenient sub-command for invoking kube-aws on a specific test cluster

* Start etcd health-checking only after etcd is tried to be started
or we'll end up with health-checking doesn't work when etcd failed to start at all

.gitignore

@@ -3,6 +3,7 @@
 /e2e/assets
 *~
 /core/*/config/templates.go
+/core/*/config/files.go
 .idea/
 .envrc
 coverage.txt

codegen/files_gen.go

@@ -0,0 +1,95 @@
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"text/template"
+	"time"
+)
+
+type Entry struct {
+	Filename string
+	VarName  string
+}
+
+type Data struct {
+	Vars []Var
+	Now  time.Time
+}
+
+type Var struct {
+	Name string
+	Data string
+}
+
+var tmpl = template.Must(template.New("files.go").Parse(`package config
+
+// This file was generated by files_gen.go. DO NOT EDIT by hand.
+//
+// Last generated at {{ .Now }}.
+
+var (
+{{ range $i, $var := .Vars }}	{{ $var.Name }} = _{{ $var.Name }}
+{{ end }}
+)
+
+var (
+{{ range $i, $var := .Vars }}	_{{ $var.Name }} = {{ $var.Data  }}{{ end }}
+)
+`))
+
+func toGoByteSlice(sli []byte) string {
+	buff := new(bytes.Buffer)
+	fmt.Fprintf(buff, "[]byte{\n")
+	for i, b := range sli {
+		if i%10 == 0 {
+			fmt.Fprintf(buff, "\t%#x,", b)
+		} else {
+			fmt.Fprintf(buff, " %#x,", b)
+		}
+		if (i+1)%10 == 0 {
+			fmt.Fprintln(buff)
+		}
+	}
+	fmt.Fprintf(buff, "\n}\n")
+	return buff.String()
+}
+
+func main() {
+	entries := []Entry{}
+	args := os.Args[1:]
+	for _, arg := range args {
+		parts := strings.Split(arg, "=")
+		varname, filename := parts[0], parts[1]
+		entry := Entry{
+			Filename: filename,
+			VarName:  varname,
+		}
+		entries = append(entries, entry)
+	}
+
+	vars := make([]Var, len(entries))
+	for i, file := range entries {
+		data, err := ioutil.ReadFile(file.Filename)
+		if err != nil {
+			log.Fatal(err)
+		}
+		vars[i] = Var{file.VarName, toGoByteSlice(data)}
+	}
+
+	f, err := os.OpenFile("files.go", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+	data := Data{vars, time.Now().UTC()}
+	if err := tmpl.Execute(f, data); err != nil {
+		log.Fatal("Failed to render template:", err)
+	}
+}

core/controlplane/config/config.go

@@ -2,6 +2,8 @@ package config
 
 //go:generate go run ../../../codegen/templates_gen.go CloudConfigController=cloud-config-controller CloudConfigWorker=cloud-config-worker CloudConfigEtcd=cloud-config-etcd DefaultClusterConfig=cluster.yaml KubeConfigTemplate=kubeconfig.tmpl StackTemplateTemplate=stack-template.json
 //go:generate gofmt -w templates.go
+//go:generate go run ../../../codegen/files_gen.go Etcdadm=../../../etcdadm/etcdadm
+//go:generate gofmt -w files.go
 
 import (
 	"errors"
@@ -17,6 +19,7 @@ import (
 	"github.com/kubernetes-incubator/kube-aws/cfnresource"
 	"github.com/kubernetes-incubator/kube-aws/coreos/amiregistry"
 	"github.com/kubernetes-incubator/kube-aws/filereader/userdatatemplate"
+	"github.com/kubernetes-incubator/kube-aws/gzipcompressor"
 	"github.com/kubernetes-incubator/kube-aws/model"
 	"github.com/kubernetes-incubator/kube-aws/model/derived"
 	"github.com/kubernetes-incubator/kube-aws/netutil"
@@ -1077,6 +1080,11 @@ func (c Cluster) NestedStackName() string {
 	return strings.Title(strings.Replace(c.StackName(), "-", "", -1))
 }
 
+// Etcdadm returns the content of the etcdadm script to be embedded into cloud-config-etcd
+func (c *Config) Etcdadm() (string, error) {
+	return gzipcompressor.CompressData(Etcdadm)
+}
+
 func (c Cluster) valid() error {
 	validClusterNaming := regexp.MustCompile("^[a-zA-Z0-9-:]+$")
 	if !validClusterNaming.MatchString(c.ClusterName) {
@@ -1171,7 +1179,7 @@ func (c Cluster) valid() error {
 
 	clusterNamePlaceholder := "<my-cluster-name>"
 	nestedStackNamePlaceHolder := "<my-nested-stack-name>"
-	replacer := strings.NewReplacer(clusterNamePlaceholder, "", nestedStackNamePlaceHolder, "")
+	replacer := strings.NewReplacer(clusterNamePlaceholder, "", nestedStackNamePlaceHolder, c.StackName())
 	simulatedLcName := fmt.Sprintf("%s-%s-1N2C4K3LLBEDZ-%sLC-BC2S9P3JG2QD", clusterNamePlaceholder, nestedStackNamePlaceHolder, c.Controller.LogicalName())
 	limit := 63 - len(replacer.Replace(simulatedLcName))
 	if c.Experimental.AwsNodeLabels.Enabled && len(c.ClusterName) > limit {
@@ -1465,11 +1473,25 @@ func (c ControllerSettings) Valid() error {
 	return nil
 }
 
+// Valid returns an error when there's any user error in the `etcd` settings
 func (e EtcdSettings) Valid() error {
 	if !e.Etcd.DataVolume.Encrypted && e.Etcd.KMSKeyARN() != "" {
 		return errors.New("`etcd.kmsKeyArn` can only be specified when `etcdDataVolumeEncrypted` is enabled")
 	}
 
+	if e.Etcd.Version().Is3() {
+		if e.Etcd.DisasterRecovery.Automated && !e.Etcd.Snapshot.Automated {
+			return errors.New("`etcd.disasterRecovery.automated` is set to true but `etcd.snapshot.automated` is not - automated disaster recovery requires snapshot to be also automated")
+		}
+	} else {
+		if e.Etcd.DisasterRecovery.Automated {
+			return errors.New("`etcd.disasterRecovery.automated` is set to true for enabling automated disaster recovery. However the feature is available only for etcd version 3")
+		}
+		if e.Etcd.Snapshot.Automated {
+			return errors.New("`etcd.snapshot.automated` is set to true for enabling automated snapshot. However the feature is available only for etcd version 3")
+		}
+	}
+
 	return nil
 }
 

core/controlplane/config/stack_config.go

@@ -6,6 +6,7 @@ import (
 	"github.com/kubernetes-incubator/kube-aws/filereader/jsontemplate"
 	"github.com/kubernetes-incubator/kube-aws/fingerprint"
 	"net/url"
+	"strings"
 )
 
 type StackConfig struct {
@@ -87,6 +88,30 @@ func (c *StackConfig) UserDataEtcdFileName() string {
 	return "userdata-etcd-" + fingerprint.SHA256(c.UserDataEtcd)
 }
 
+func (c *StackConfig) EtcdSnapshotsS3Path() (string, error) {
+	s3uri, err := url.Parse(c.S3URI)
+	if err != nil {
+		return "", fmt.Errorf("Error in EtcdSnapshotsS3Path : %v", err)
+	}
+	return fmt.Sprintf("%s%s/etcd-snapshots", s3uri.Host, s3uri.Path), nil
+}
+
+func (c *StackConfig) EtcdSnapshotsS3Bucket() (string, error) {
+	s3uri, err := url.Parse(c.S3URI)
+	if err != nil {
+		return "", fmt.Errorf("Error in EtcdSnapshotsS3Bucket : %v", err)
+	}
+	return s3uri.Host, nil
+}
+
+func (c *StackConfig) EtcdSnapshotsS3Prefix() (string, error) {
+	s3uri, err := url.Parse(c.S3URI)
+	if err != nil {
+		return "", fmt.Errorf("Error in EtcdSnapshotsS3Prefix : %v", err)
+	}
+	return strings.TrimLeft(s3uri.Path, "/"), nil
+}
+
 func (c *StackConfig) ValidateUserData() error {
 	err := userdatavalidation.Execute([]userdatavalidation.Entry{
 		{Name: "UserDataWorker", Content: c.UserDataWorker},

core/controlplane/config/templates/cloud-config-controller

@@ -1060,6 +1060,9 @@ write_files:
           - --allow-privileged=true
           - --service-cluster-ip-range={{.ServiceCIDR}}
           - --secure-port=443
+          {{if .Etcd.Version.Is3}}
+          - --storage-backend=etcd3
+          {{end}}
           - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
           {{ if .AuthTokensConfig.HasTokens }}
           - --token-auth-file=/etc/kubernetes/auth/tokens.csv