Merge pull request #2780 from openziti/add-controller-ha-shutdown

Shutdown raft when controller shuts down

CHANGELOG.md

@@ -178,26 +178,28 @@ file, enrollment should handle initializing the value appropriately.
 
 ## Component Updates and Bug Fixes
 
-* github.com/openziti/agent: [v1.0.23 -> v1.0.24](https://github.com/openziti/agent/compare/v1.0.23...v1.0.24)
-* github.com/openziti/channel/v3: [v3.0.26 -> v3.0.29](https://github.com/openziti/channel/compare/v3.0.26...v3.0.29)
+* github.com/openziti/agent: [v1.0.23 -> v1.0.25](https://github.com/openziti/agent/compare/v1.0.23...v1.0.25)
+* github.com/openziti/channel/v3: [v3.0.26 -> v3.0.31](https://github.com/openziti/channel/compare/v3.0.26...v3.0.31)
 * github.com/openziti/edge-api: [v0.26.38 -> v0.26.39](https://github.com/openziti/edge-api/compare/v0.26.38...v0.26.39)
 * github.com/openziti/foundation/v2: [v2.0.56 -> v2.0.57](https://github.com/openziti/foundation/compare/v2.0.56...v2.0.57)
-* github.com/openziti/identity: [v1.0.94 -> v1.0.97](https://github.com/openziti/identity/compare/v1.0.94...v1.0.97)
+* github.com/openziti/identity: [v1.0.94 -> v1.0.98](https://github.com/openziti/identity/compare/v1.0.94...v1.0.98)
 * github.com/openziti/metrics: [v1.2.65 -> v1.2.66](https://github.com/openziti/metrics/compare/v1.2.65...v1.2.66)
 * github.com/openziti/runzmd: [v1.0.59 -> v1.0.63](https://github.com/openziti/runzmd/compare/v1.0.59...v1.0.63)
 * github.com/openziti/sdk-golang: [v0.23.44 -> v0.24.1](https://github.com/openziti/sdk-golang/compare/v0.23.44...v0.24.1)
     * [Issue #673](https://github.com/openziti/sdk-golang/issues/673) - Add license check to GH workflow
     * [Issue #663](https://github.com/openziti/sdk-golang/issues/663) - Add API to allow controlling proxying connections to controllers and routers.
     * [Issue #659](https://github.com/openziti/sdk-golang/issues/659) - E2E encryption can encounter ordering issues with high-volume concurrent writes
 
-* github.com/openziti/storage: [v0.3.15 -> v0.4.2](https://github.com/openziti/storage/compare/v0.3.15...v0.4.2)
+* github.com/openziti/secretstream: [v0.1.28 -> v0.1.29](https://github.com/openziti/secretstream/compare/v0.1.28...v0.1.29)
+* github.com/openziti/storage: [v0.3.15 -> v0.4.3](https://github.com/openziti/storage/compare/v0.3.15...v0.4.3)
     * [Issue #94](https://github.com/openziti/storage/issues/94) - Snapshots aren't working correctly
 
-* github.com/openziti/transport/v2: [v2.0.159 -> v2.0.160](https://github.com/openziti/transport/compare/v2.0.159...v2.0.160)
+* github.com/openziti/transport/v2: [v2.0.159 -> v2.0.162](https://github.com/openziti/transport/compare/v2.0.159...v2.0.162)
 * github.com/openziti/xweb/v2: [v2.1.3 -> v2.2.1](https://github.com/openziti/xweb/compare/v2.1.3...v2.2.1)
     * [Issue #18](https://github.com/openziti/xweb/issues/18) - verify advertised host/ip has a certificate defined in the identity block
 
 * github.com/openziti/ziti: [v1.3.3 -> v1.4.0](https://github.com/openziti/ziti/compare/v1.3.3...v1.4.0)
+    * [Issue #2774](https://github.com/openziti/ziti/issues/2774) - API Session Certificate SPIFFE IDs fail validation in Routers
     * [Issue #2672](https://github.com/openziti/ziti/issues/2672) - [Bug] Posture check PUT method doesn't update nested structures but works fine with PATCH
     * [Issue #2668](https://github.com/openziti/ziti/issues/2668) - [Feature Request] Filterable field for posture check type
     * [Issue #2681](https://github.com/openziti/ziti/issues/2681) - Support specifying which token to use on external jwt signers
@@ -220,6 +222,7 @@ file, enrollment should handle initializing the value appropriately.
     * [Issue #2546](https://github.com/openziti/ziti/issues/2546) - Use consistent terminology for HA
     * [Issue #2713](https://github.com/openziti/ziti/issues/2713) - Routers with no edge components shouldn't subscribe to RDM updates
 
+
 # Release 1.3.3
 
 ## What's New

controller/controller.go

@@ -479,6 +479,12 @@ func (c *Controller) Shutdown() {
 		}
 
 		go c.xweb.Shutdown()
+
+		if c.raftController != nil {
+			if err := c.raftController.Shutdown(); err != nil {
+				pfxlog.Logger().WithError(err).Error("failed to shutdown raft")
+			}
+		}
 	}
 }
 

controller/db/api_session_store.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"go.etcd.io/bbolt"
+	bolterrors "go.etcd.io/bbolt/errors"
 	"strings"
 	"time"
 )
@@ -174,7 +175,7 @@ func (store *apiSessionStoreImpl) cleanupSessions(db boltz.Db, name string, apiS
 		indexPath := []string{RootBucket, boltz.IndexesBucket, EntityTypeApiSessions, EntityTypeSessions}
 		if bucket := boltz.Path(ctx.Tx(), indexPath...); bucket != nil {
 			if err := bucket.DeleteBucket(apiSessionId); err != nil {
-				if !errors.Is(err, bbolt.ErrBucketNotFound) {
+				if !errors.Is(err, bolterrors.ErrBucketNotFound) {
 					logger.WithError(err).
 						Error("error deleting for api session index associated to an api session during onEventualDelete")
 				}

controller/raft/fsm.go

@@ -30,6 +30,7 @@ import (
 	event2 "github.com/openziti/ziti/controller/event"
 	"github.com/sirupsen/logrus"
 	"go.etcd.io/bbolt"
+	bbolterrors "go.etcd.io/bbolt/errors"
 	"io"
 	"os"
 	"path"
@@ -91,6 +92,10 @@ func (self *BoltDbFsm) Init() error {
 	return nil
 }
 
+func (self *BoltDbFsm) Close() error {
+	return self.db.Close()
+}
+
 func (self *BoltDbFsm) GetDb() boltz.Db {
 	return self.db
 }
@@ -155,7 +160,7 @@ func (self *BoltDbFsm) storeConfigurationInRaft(index uint64, servers []raft.Ser
 func (self *BoltDbFsm) storeServers(tx *bbolt.Tx, servers []raft.Server) error {
 	raftBucket := boltz.GetOrCreatePath(tx, db.RootBucket, db.MetadataBucket)
 	if err := raftBucket.DeleteBucket([]byte(ServersBucket)); err != nil {
-		if !errors.Is(err, bbolt.ErrBucketNotFound) {
+		if !errors.Is(err, bbolterrors.ErrBucketNotFound) {
 			return err
 		}
 	}

controller/raft/mesh/mesh.go

@@ -363,6 +363,13 @@ func (self *impl) Close() error {
 	if self.closed.CompareAndSwap(false, true) {
 		close(self.closeNotify)
 	}
+
+	for _, p := range self.GetPeers() {
+		if err := p.Channel.Close(); err != nil {
+			pfxlog.Logger().WithError(err).Error("failed to close ctrl mesh peer channel")
+		}
+	}
+
 	return nil
 }
 
@@ -382,6 +389,10 @@ func (self *impl) Accept() (net.Conn, error) {
 }
 
 func (self *impl) Dial(address raft.ServerAddress, timeout time.Duration) (net.Conn, error) {
+	if self.closed.Load() {
+		return nil, errors.New("ctrl mesh is closed")
+	}
+
 	log := pfxlog.Logger().WithField("address", address)
 	log.Info("dialing raft peer channel")
 	peer, err := self.GetOrConnectPeer(string(address), timeout)

controller/raft/raft.go

@@ -141,6 +141,7 @@ type Controller struct {
 	Mesh                       mesh.Mesh
 	Raft                       *raft.Raft
 	Fsm                        *BoltDbFsm
+	raftStore                  *raftboltdb.BoltStore
 	bootstrapped               atomic.Bool
 	clusterLock                sync.Mutex
 	closeNotify                <-chan struct{}
@@ -545,7 +546,8 @@ func (self *Controller) Init() error {
 
 	// Create the log store and stable store.
 	raftBoltFile := path.Join(raftConfig.DataDir, "raft.db")
-	boltDbStore, err := raftboltdb.NewBoltStore(raftBoltFile)
+	var err error
+	self.raftStore, err = raftboltdb.NewBoltStore(raftBoltFile)
 	if err != nil {
 		logrus.WithError(err).Error("failed to initialize raft bolt storage")
 		return err
@@ -578,7 +580,7 @@ func (self *Controller) Init() error {
 	raftTransport := raft.NewNetworkTransportWithLogger(self.Mesh, 3, 10*time.Second, raftConfig.Logger)
 
 	if raftConfig.Recover {
-		err := raft.RecoverCluster(conf, self.Fsm, boltDbStore, boltDbStore, snapshotStore, raftTransport, raft.Configuration{
+		err := raft.RecoverCluster(conf, self.Fsm, self.raftStore, self.raftStore, snapshotStore, raftTransport, raft.Configuration{
 			Servers: []raft.Server{
 				{ID: conf.LocalID, Address: localAddr},
 			},
@@ -591,7 +593,7 @@ func (self *Controller) Init() error {
 		os.Exit(0)
 	}
 
-	r, err := raft.NewRaft(conf, self.Fsm, boltDbStore, boltDbStore, snapshotStore, raftTransport)
+	r, err := raft.NewRaft(conf, self.Fsm, self.raftStore, self.raftStore, snapshotStore, raftTransport)
 	if err != nil {
 		return errors.Wrap(err, "failed to initialise raft")
 	}
@@ -938,6 +940,36 @@ func (self *Controller) addEventsHandlers() {
 	})
 }
 
+func (self *Controller) Shutdown() error {
+	var errs errorz.MultipleErrors
+
+	if self.Raft != nil {
+		if err := self.Raft.Shutdown().Error(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if self.Fsm != nil {
+		if err := self.Fsm.Close(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if self.raftStore != nil {
+		if err := self.raftStore.Close(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if self.Mesh != nil {
+		if err := self.Mesh.Close(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	return errs.ToError()
+}
+
 type MigrationManager interface {
 	ValidateMigrationEnvironment() error
 	TryInitializeRaftFromBoltDb() error

go.mod

@@ -52,19 +52,19 @@ require (
 	github.com/miekg/dns v1.1.63
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/natefinch/lumberjack v2.0.0+incompatible
-	github.com/openziti/agent v1.0.24
-	github.com/openziti/channel/v3 v3.0.29
+	github.com/openziti/agent v1.0.25
+	github.com/openziti/channel/v3 v3.0.31
 	github.com/openziti/cobra-to-md v1.0.1
 	github.com/openziti/edge-api v0.26.39
 	github.com/openziti/foundation/v2 v2.0.57
-	github.com/openziti/identity v1.0.97
+	github.com/openziti/identity v1.0.98
 	github.com/openziti/jwks v1.0.6
 	github.com/openziti/metrics v1.2.66
 	github.com/openziti/runzmd v1.0.63
 	github.com/openziti/sdk-golang v0.24.1
-	github.com/openziti/secretstream v0.1.28
-	github.com/openziti/storage v0.4.2
-	github.com/openziti/transport/v2 v2.0.160
+	github.com/openziti/secretstream v0.1.29
+	github.com/openziti/storage v0.4.3
+	github.com/openziti/transport/v2 v2.0.162
 	github.com/openziti/x509-claims v1.0.3
 	github.com/openziti/xweb/v2 v2.2.1
 	github.com/openziti/ziti-db-explorer v1.1.3
@@ -83,15 +83,15 @@ require (
 	github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/zitadel/oidc/v2 v2.12.2
-	go.etcd.io/bbolt v1.3.11
+	go.etcd.io/bbolt v1.4.0
 	go4.org v0.0.0-20180809161055-417644f6feb5
-	golang.org/x/crypto v0.32.0
+	golang.org/x/crypto v0.33.0
 	golang.org/x/exp v0.0.0-20241210194714-1829a127f884
-	golang.org/x/net v0.34.0
+	golang.org/x/net v0.35.0
 	golang.org/x/oauth2 v0.25.0
-	golang.org/x/sync v0.10.0
+	golang.org/x/sync v0.11.0
 	golang.org/x/sys v0.30.0
-	golang.org/x/text v0.21.0
+	golang.org/x/text v0.22.0
 	google.golang.org/protobuf v1.36.5
 	gopkg.in/AlecAivazis/survey.v1 v1.8.8
 	gopkg.in/go-jose/go-jose.v2 v2.6.3
@@ -138,7 +138,7 @@ require (
 	github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/iancoleman/strcase v0.1.3 // indirect
+	github.com/iancoleman/strcase v0.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
@@ -166,7 +166,7 @@ require (
 	github.com/parallaxsecond/parsec-client-go v0.0.0-20221025095442-f0a77d263cf9 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pion/dtls/v3 v3.0.4 // indirect
-	github.com/pion/logging v0.2.2 // indirect
+	github.com/pion/logging v0.2.3 // indirect
 	github.com/pion/transport/v3 v3.0.7 // indirect
 	github.com/pkg/term v1.2.0-beta.2 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect