Endpoint management updates

.gitignore

@@ -34,3 +34,4 @@ go.work.sum
 doc/ha/pki
 doc/ha/data
 simple-transfer-*/
+etc/endpoints

CHANGELOG.md

@@ -9,6 +9,7 @@
     * Moved `ziti ops verify-network` to `ziti ops verify network`
     * Moved `ziti ops verify traffic` to `ziti ops verify traffic`
     * Added `ziti ops verify ext-jwt-signer oidc` to help users with configuring OIDC external auth 
+* Router Controller Endpoint Change#s
 * Bug fixes
 
 ## Event Doc and Consistency
@@ -118,12 +119,59 @@ will move the database into place and then shutdown, expecting to be restarted.
 because there is caching in various places and restartingi makes sure that everything is
 coherent with the changes database.
 
+## Router Controller Endpoint Updates
+
+### Endpoints File Config
+
+The config setting for controller the endpoints file location has changed.
+
+It was:
+
+```
+ctrl:
+  dataDir: /path/to/dir
+```
+
+The endpoints file would live in that directory but always be called endpoints.
+
+This is replaced by a more flexible `endpointsFile`.
+
+```
+ctrl:
+  endpointsFile: /path/to/endpoints.file
+```
+
+The default location is unchanged, which is a file named `endpoints` in the same
+directory as the router config file.
+
+### Enrollment
+
+The router enrollment will now contain the set of known controllers at the time
+the router as enrollled. This also works for standalone controllers, as long as
+the `advertiseAddress` settings is set.
+
+Example
+
+```
+ctrl:
+  listener: tls:0.0.0.0:6262
+  options:
+    advertiseAddress: tls:ctrl1.ziti.example.com
+```
+
+This means that the controller no longer needs to be set manually in the config 
+file, enrollment should handle initializing the value appropriately.
+
 ## Component Updates and Bug Fixes
 
 * github.com/openziti/storage: [v0.3.15 -> v0.4.1](https://github.com/openziti/storage/compare/v0.3.15...v0.4.1)
     * [Issue #94](https://github.com/openziti/storage/issues/94) - Snapshots aren't working correctly
 
 * github.com/openziti/ziti: [v1.3.3 -> v1.4.0](https://github.com/openziti/ziti/compare/v1.3.3...v1.4.0)
+    * [Issue #2724](https://github.com/openziti/ziti/issues/2724) - Allow configuring endpoints file full path instead of directory
+    * [Issue #2728](https://github.com/openziti/ziti/issues/2728) - Write initial router endpoints file based on ctrls in JWT
+    * [Issue #2108](https://github.com/openziti/ziti/issues/2108) - Add `ctrls` property to non-ha router enrollment
+    * [Issue #2729](https://github.com/openziti/ziti/issues/2729) - Enrollment doesn't contain controller which created the enrollment
     * [Issue #2549](https://github.com/openziti/ziti/issues/2549) - Handle Index Non HA to HA Transitions During Upgrades
     * [Issue #2649](https://github.com/openziti/ziti/issues/2649) - Make restoring an HA cluster from a DB backup easier
     * [Issue #2707](https://github.com/openziti/ziti/issues/2707) - Ensure database restores work with RDM enabled routers

controller/model/enrollment_model.go

@@ -59,15 +59,27 @@ func (entity *Enrollment) FillJwtInfoWithExpiresAt(env Env, subject string, expi
 		entity.Token = uuid.New().String()
 	}
 
-	peerControllers := env.GetPeerControllerAddresses()
+	ctrls, err := env.GetManagers().Controller.BaseList("true limit none")
+	if err != nil {
+		return fmt.Errorf("could not get controllers to populate JWT %w", err)
+	}
 
-	for i, addr := range peerControllers {
-		peerControllers[i] = "https://" + addr
+	var controllers []string
+	for _, ctrl := range ctrls.Entities {
+		controllers = append(controllers, ctrl.CtrlAddress)
+	}
+
+	if len(controllers) == 0 {
+		if options := env.GetConfig().Ctrl.Options; options != nil {
+			if advertiseAddr := env.GetConfig().Ctrl.Options.AdvertiseAddress; advertiseAddr != nil {
+				controllers = append(controllers, (*advertiseAddr).String())
+			}
+		}
 	}
 
 	enrollmentClaims := &ziti.EnrollmentClaims{
 		EnrollmentMethod: entity.Method,
-		Controllers:      peerControllers,
+		Controllers:      controllers,
 		RegisteredClaims: jwt.RegisteredClaims{
 			Audience:  []string{""},
 			ExpiresAt: &jwt.NumericDate{Time: expiresAt},

doc/ha/create-pki.sh

@@ -7,14 +7,23 @@ ziti pki create intermediate --pki-root ./pki --ca-name ca --intermediate-file c
 # Create the controller 1 server cert
 ziti pki create server --pki-root ./pki --ca-name ctrl1 --dns localhost --ip 127.0.0.1 --server-name ctrl1 --spiffe-id 'controller/ctrl1'
 
+# Create the controller 1 server cert
+ziti pki create client --pki-root ./pki --ca-name ctrl1 --client-name ctrl1 --spiffe-id 'controller/ctrl1'
+
 # Create the controller 2 intermediate/signing cert
 ziti pki create intermediate --pki-root ./pki --ca-name ca --intermediate-file ctrl2 --intermediate-name 'Controller Two Signing Cert'
 
 # Create the controller 2 server cert
 ziti pki create server --pki-root ./pki --ca-name ctrl2 --dns localhost --ip 127.0.0.1 --server-name ctrl2 --spiffe-id 'controller/ctrl2'
 
+# Create the controller 2 client cert
+ziti pki create client --pki-root ./pki --ca-name ctrl2 --client-name ctrl2 --spiffe-id 'controller/ctrl2'
+
 # Create the controller 3 intermediate/signing cert
 ziti pki create intermediate --pki-root ./pki --ca-name ca --intermediate-file ctrl3 --intermediate-name 'Controller Three Signing Cert'
 
 # Create the controller 3 server cert
 ziti pki create server --pki-root ./pki --ca-name ctrl3 --dns localhost --ip 127.0.0.1 --server-name ctrl3 --spiffe-id 'controller/ctrl3'
+
+# Create the controller 3 client cert
+ziti pki create client --pki-root ./pki --ca-name ctrl3 --client-name ctrl3 --spiffe-id 'controller/ctrl3'

etc/ctrl.with.edge.yml

@@ -61,6 +61,8 @@ identity:
 ctrl:
   listener: tls:127.0.0.1:6262
   options:
+    advertiseAddress: tls:127.0.0.1:6262
+
     # (optional) settings
     # set the maximum number of connect requests that are buffered and waiting to be acknowledged (1 to 5000, default 1000)
     #maxQueuedConnects:      50
@@ -82,7 +84,7 @@ ctrl:
 events:
   jsonLogger:
     subscriptions:
-#      - type: apiSession
+      - type: apiSession
 #      - type: circuit
 #      - type: connect
 #      - type: sdk

router/config.go

@@ -129,6 +129,7 @@ func SetConfigMapFlags(cfgmap map[interface{}]interface{}, flags map[string]*pfl
 }
 
 type Config struct {
+	IdConfig       *identity.Config
 	Id             *identity.TokenId
 	EnableDebugOps bool
 	Forwarder      *forwarder.Options
@@ -149,7 +150,7 @@ type Config struct {
 		LocalBinding          string
 		DefaultRequestTimeout time.Duration
 		Options               *channel.Options
-		DataDir               string
+		EndpointsFile         string
 		Heartbeats            env.HeartbeatOptions
 		StartupTimeout        time.Duration
 		RateLimit             command.AdaptiveRateLimiterConfig
@@ -182,6 +183,7 @@ type Config struct {
 	ConnectEvents env.ConnectEventsConfig
 	Proxy         *transport.ProxyConfiguration
 	Plugins       []string
+	Edge          *EdgeConfig
 	src           map[interface{}]interface{}
 	path          string
 }
@@ -369,6 +371,10 @@ func (c *UpdatableAddress) UnmarshalYAML(value *yaml3.Node) error {
 }
 
 func LoadConfig(path string) (*Config, error) {
+	return LoadConfigWithOptions(path, true)
+}
+
+func LoadConfigWithOptions(path string, loadIdentity bool) (*Config, error) {
 	cfgmap, err := LoadConfigMap(path)
 
 	if err != nil {
@@ -391,13 +397,17 @@ func LoadConfig(path string) (*Config, error) {
 		return nil, fmt.Errorf("unable to load identity: %v", err)
 	}
 
-	if id, err := identity.LoadIdentity(*identityConfig); err != nil {
-		return nil, fmt.Errorf("unable to load identity (%w)", err)
-	} else {
-		cfg.Id = identity.NewIdentity(id)
+	cfg.IdConfig = identityConfig
+
+	if loadIdentity {
+		if id, err := identity.LoadIdentity(*identityConfig); err != nil {
+			return nil, fmt.Errorf("unable to load identity (%w)", err)
+		} else {
+			cfg.Id = identity.NewIdentity(id)
 
-		if err := cfg.Id.WatchFiles(); err != nil {
-			pfxlog.Logger().Warn("could not enable file watching on identity: %w", err)
+			if err := cfg.Id.WatchFiles(); err != nil {
+				pfxlog.Logger().Warn("could not enable file watching on identity: %w", err)
+			}
 		}
 	}
 
@@ -539,10 +549,10 @@ func LoadConfig(path string) (*Config, error) {
 					return nil, errors.Wrap(err, "invalid value for ctrl.startupTimeout")
 				}
 			}
-			if value, found := submap["dataDir"]; found {
-				cfg.Ctrl.DataDir = value.(string)
+			if value, found := submap["endpointsFile"]; found {
+				cfg.Ctrl.EndpointsFile = value.(string)
 			} else {
-				cfg.Ctrl.DataDir = filepath.Dir(cfg.path)
+				cfg.Ctrl.EndpointsFile = filepath.Join(filepath.Dir(cfg.path), "endpoints")
 			}
 			if err = cfg.loadCtrlRateLimiterConfig(submap); err != nil {
 				return nil, err
@@ -865,6 +875,11 @@ func LoadConfig(path string) (*Config, error) {
 		cfg.ConnectEvents.MaxQueuedEvents = MaxConnectEventsMaxQueuedEvents
 	}
 
+	cfg.Edge = NewEdgeConfig(cfg)
+	if err = cfg.Edge.LoadEdgeConfigFromMap(cfgmap, loadIdentity); err != nil {
+		return nil, err
+	}
+
 	return cfg, nil
 }
 
@@ -905,6 +920,21 @@ func (c *Config) loadCtrlRateLimiterConfig(cfgmap map[interface{}]interface{}) e
 	return nil
 }
 
+func (c *Config) SaveControllerEndpoints(endpoints []string) error {
+	endpointsFile := c.Ctrl.EndpointsFile
+
+	configData := map[string]interface{}{
+		"endpoints": endpoints,
+	}
+
+	if data, err := yaml.Marshal(configData); err != nil {
+		return fmt.Errorf("unable to marshal updated controller endpoints to yaml (%w)", err)
+	} else if err = os.WriteFile(endpointsFile, data, 0600); err != nil {
+		return fmt.Errorf("unable to write updated controller endpoints to file '%s' (%w)", c.Ctrl.EndpointsFile, err)
+	}
+	return nil
+}
+
 func LoadIdentityConfigFromMap(cfgmap map[interface{}]interface{}) (*identity.Config, error) {
 	if value, found := cfgmap["identity"]; found {
 		subMap := value.(map[interface{}]interface{})