From fef5327e142b484f692df72b3638f157d0a02a5d Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Mon, 3 Feb 2025 16:24:32 -0500 Subject: [PATCH] Endpoint management updates * Allow configuring endpoints file full path instead of directory. Fixes #2724 * Write initial router endpoints file based on ctrls in JWT. Fixes #2728 * Add ctrls property to non-ha router enrollment. Fixes #2108 * Enrollment doesn't contain controller which created the enrollment. Fixes #2729 --- .gitignore | 1 + CHANGELOG.md | 48 ++++++++++ controller/model/enrollment_model.go | 20 ++++- doc/ha/create-pki.sh | 9 ++ etc/ctrl.with.edge.yml | 4 +- router/config.go | 50 ++++++++--- .../edgerouter/config.go => configedge.go} | 88 +++++++++---------- router/enroll/csr.go | 4 +- router/enroll/enroll.go | 39 +++----- router/internal/apiproxy/proxy.go | 4 +- router/internal/edgerouter/types.go | 37 -------- router/router.go | 41 ++++----- router/router_test.go | 11 +-- router/xgress_edge/certchecker.go | 8 +- router/xgress_edge/certchecker_test.go | 4 +- router/xgress_edge/factory.go | 22 ++--- tests/context.go | 10 +-- tests/endpoints | 2 +- ziti/router/enrollgw.go | 15 ++-- 19 files changed, 229 insertions(+), 188 deletions(-) rename router/{internal/edgerouter/config.go => configedge.go} (87%) delete mode 100644 router/internal/edgerouter/types.go diff --git a/.gitignore b/.gitignore index 008d32d53..dcd34220a 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ go.work.sum doc/ha/pki doc/ha/data simple-transfer-*/ +etc/endpoints diff --git a/CHANGELOG.md b/CHANGELOG.md index a56ad02ef..9fba7ed64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * Moved `ziti ops verify-network` to `ziti ops verify network` * Moved `ziti ops verify traffic` to `ziti ops verify traffic` * Added `ziti ops verify ext-jwt-signer oidc` to help users with configuring OIDC external auth +* Router Controller Endpoint Change#s * Bug fixes ## Event Doc and Consistency @@ -118,12 +119,59 @@ will move the database into place and then shutdown, expecting to be restarted. because there is caching in various places and restartingi makes sure that everything is coherent with the changes database. +## Router Controller Endpoint Updates + +### Endpoints File Config + +The config setting for controller the endpoints file location has changed. + +It was: + +``` +ctrl: + dataDir: /path/to/dir +``` + +The endpoints file would live in that directory but always be called endpoints. + +This is replaced by a more flexible `endpointsFile`. + +``` +ctrl: + endpointsFile: /path/to/endpoints.file +``` + +The default location is unchanged, which is a file named `endpoints` in the same +directory as the router config file. + +### Enrollment + +The router enrollment will now contain the set of known controllers at the time +the router as enrollled. This also works for standalone controllers, as long as +the `advertiseAddress` settings is set. + +Example + +``` +ctrl: + listener: tls:0.0.0.0:6262 + options: + advertiseAddress: tls:ctrl1.ziti.example.com +``` + +This means that the controller no longer needs to be set manually in the config +file, enrollment should handle initializing the value appropriately. + ## Component Updates and Bug Fixes * github.com/openziti/storage: [v0.3.15 -> v0.4.1](https://github.com/openziti/storage/compare/v0.3.15...v0.4.1) * [Issue #94](https://github.com/openziti/storage/issues/94) - Snapshots aren't working correctly * github.com/openziti/ziti: [v1.3.3 -> v1.4.0](https://github.com/openziti/ziti/compare/v1.3.3...v1.4.0) + * [Issue #2724](https://github.com/openziti/ziti/issues/2724) - Allow configuring endpoints file full path instead of directory + * [Issue #2728](https://github.com/openziti/ziti/issues/2728) - Write initial router endpoints file based on ctrls in JWT + * [Issue #2108](https://github.com/openziti/ziti/issues/2108) - Add `ctrls` property to non-ha router enrollment + * [Issue #2729](https://github.com/openziti/ziti/issues/2729) - Enrollment doesn't contain controller which created the enrollment * [Issue #2549](https://github.com/openziti/ziti/issues/2549) - Handle Index Non HA to HA Transitions During Upgrades * [Issue #2649](https://github.com/openziti/ziti/issues/2649) - Make restoring an HA cluster from a DB backup easier * [Issue #2707](https://github.com/openziti/ziti/issues/2707) - Ensure database restores work with RDM enabled routers diff --git a/controller/model/enrollment_model.go b/controller/model/enrollment_model.go index 639664888..5251ee963 100644 --- a/controller/model/enrollment_model.go +++ b/controller/model/enrollment_model.go @@ -59,15 +59,27 @@ func (entity *Enrollment) FillJwtInfoWithExpiresAt(env Env, subject string, expi entity.Token = uuid.New().String() } - peerControllers := env.GetPeerControllerAddresses() + ctrls, err := env.GetManagers().Controller.BaseList("true limit none") + if err != nil { + return fmt.Errorf("could not get controllers to populate JWT %w", err) + } - for i, addr := range peerControllers { - peerControllers[i] = "https://" + addr + var controllers []string + for _, ctrl := range ctrls.Entities { + controllers = append(controllers, ctrl.CtrlAddress) + } + + if len(controllers) == 0 { + if options := env.GetConfig().Ctrl.Options; options != nil { + if advertiseAddr := env.GetConfig().Ctrl.Options.AdvertiseAddress; advertiseAddr != nil { + controllers = append(controllers, (*advertiseAddr).String()) + } + } } enrollmentClaims := &ziti.EnrollmentClaims{ EnrollmentMethod: entity.Method, - Controllers: peerControllers, + Controllers: controllers, RegisteredClaims: jwt.RegisteredClaims{ Audience: []string{""}, ExpiresAt: &jwt.NumericDate{Time: expiresAt}, diff --git a/doc/ha/create-pki.sh b/doc/ha/create-pki.sh index fbcd9b921..fe6c3b5bb 100755 --- a/doc/ha/create-pki.sh +++ b/doc/ha/create-pki.sh @@ -7,14 +7,23 @@ ziti pki create intermediate --pki-root ./pki --ca-name ca --intermediate-file c # Create the controller 1 server cert ziti pki create server --pki-root ./pki --ca-name ctrl1 --dns localhost --ip 127.0.0.1 --server-name ctrl1 --spiffe-id 'controller/ctrl1' +# Create the controller 1 server cert +ziti pki create client --pki-root ./pki --ca-name ctrl1 --client-name ctrl1 --spiffe-id 'controller/ctrl1' + # Create the controller 2 intermediate/signing cert ziti pki create intermediate --pki-root ./pki --ca-name ca --intermediate-file ctrl2 --intermediate-name 'Controller Two Signing Cert' # Create the controller 2 server cert ziti pki create server --pki-root ./pki --ca-name ctrl2 --dns localhost --ip 127.0.0.1 --server-name ctrl2 --spiffe-id 'controller/ctrl2' +# Create the controller 2 client cert +ziti pki create client --pki-root ./pki --ca-name ctrl2 --client-name ctrl2 --spiffe-id 'controller/ctrl2' + # Create the controller 3 intermediate/signing cert ziti pki create intermediate --pki-root ./pki --ca-name ca --intermediate-file ctrl3 --intermediate-name 'Controller Three Signing Cert' # Create the controller 3 server cert ziti pki create server --pki-root ./pki --ca-name ctrl3 --dns localhost --ip 127.0.0.1 --server-name ctrl3 --spiffe-id 'controller/ctrl3' + +# Create the controller 3 client cert +ziti pki create client --pki-root ./pki --ca-name ctrl3 --client-name ctrl3 --spiffe-id 'controller/ctrl3' diff --git a/etc/ctrl.with.edge.yml b/etc/ctrl.with.edge.yml index 564d153f8..4dd5c7c92 100644 --- a/etc/ctrl.with.edge.yml +++ b/etc/ctrl.with.edge.yml @@ -61,6 +61,8 @@ identity: ctrl: listener: tls:127.0.0.1:6262 options: + advertiseAddress: tls:127.0.0.1:6262 + # (optional) settings # set the maximum number of connect requests that are buffered and waiting to be acknowledged (1 to 5000, default 1000) #maxQueuedConnects: 50 @@ -82,7 +84,7 @@ ctrl: events: jsonLogger: subscriptions: -# - type: apiSession + - type: apiSession # - type: circuit # - type: connect # - type: sdk diff --git a/router/config.go b/router/config.go index 651589ca2..e89df8346 100644 --- a/router/config.go +++ b/router/config.go @@ -129,6 +129,7 @@ func SetConfigMapFlags(cfgmap map[interface{}]interface{}, flags map[string]*pfl } type Config struct { + IdConfig *identity.Config Id *identity.TokenId EnableDebugOps bool Forwarder *forwarder.Options @@ -149,7 +150,7 @@ type Config struct { LocalBinding string DefaultRequestTimeout time.Duration Options *channel.Options - DataDir string + EndpointsFile string Heartbeats env.HeartbeatOptions StartupTimeout time.Duration RateLimit command.AdaptiveRateLimiterConfig @@ -182,6 +183,7 @@ type Config struct { ConnectEvents env.ConnectEventsConfig Proxy *transport.ProxyConfiguration Plugins []string + Edge *EdgeConfig src map[interface{}]interface{} path string } @@ -369,6 +371,10 @@ func (c *UpdatableAddress) UnmarshalYAML(value *yaml3.Node) error { } func LoadConfig(path string) (*Config, error) { + return LoadConfigWithOptions(path, true) +} + +func LoadConfigWithOptions(path string, loadIdentity bool) (*Config, error) { cfgmap, err := LoadConfigMap(path) if err != nil { @@ -391,13 +397,17 @@ func LoadConfig(path string) (*Config, error) { return nil, fmt.Errorf("unable to load identity: %v", err) } - if id, err := identity.LoadIdentity(*identityConfig); err != nil { - return nil, fmt.Errorf("unable to load identity (%w)", err) - } else { - cfg.Id = identity.NewIdentity(id) + cfg.IdConfig = identityConfig + + if loadIdentity { + if id, err := identity.LoadIdentity(*identityConfig); err != nil { + return nil, fmt.Errorf("unable to load identity (%w)", err) + } else { + cfg.Id = identity.NewIdentity(id) - if err := cfg.Id.WatchFiles(); err != nil { - pfxlog.Logger().Warn("could not enable file watching on identity: %w", err) + if err := cfg.Id.WatchFiles(); err != nil { + pfxlog.Logger().Warn("could not enable file watching on identity: %w", err) + } } } @@ -539,10 +549,10 @@ func LoadConfig(path string) (*Config, error) { return nil, errors.Wrap(err, "invalid value for ctrl.startupTimeout") } } - if value, found := submap["dataDir"]; found { - cfg.Ctrl.DataDir = value.(string) + if value, found := submap["endpointsFile"]; found { + cfg.Ctrl.EndpointsFile = value.(string) } else { - cfg.Ctrl.DataDir = filepath.Dir(cfg.path) + cfg.Ctrl.EndpointsFile = filepath.Join(filepath.Dir(cfg.path), "endpoints") } if err = cfg.loadCtrlRateLimiterConfig(submap); err != nil { return nil, err @@ -865,6 +875,11 @@ func LoadConfig(path string) (*Config, error) { cfg.ConnectEvents.MaxQueuedEvents = MaxConnectEventsMaxQueuedEvents } + cfg.Edge = NewEdgeConfig(cfg) + if err = cfg.Edge.LoadEdgeConfigFromMap(cfgmap, loadIdentity); err != nil { + return nil, err + } + return cfg, nil } @@ -905,6 +920,21 @@ func (c *Config) loadCtrlRateLimiterConfig(cfgmap map[interface{}]interface{}) e return nil } +func (c *Config) SaveControllerEndpoints(endpoints []string) error { + endpointsFile := c.Ctrl.EndpointsFile + + configData := map[string]interface{}{ + "endpoints": endpoints, + } + + if data, err := yaml.Marshal(configData); err != nil { + return fmt.Errorf("unable to marshal updated controller endpoints to yaml (%w)", err) + } else if err = os.WriteFile(endpointsFile, data, 0600); err != nil { + return fmt.Errorf("unable to write updated controller endpoints to file '%s' (%w)", c.Ctrl.EndpointsFile, err) + } + return nil +} + func LoadIdentityConfigFromMap(cfgmap map[interface{}]interface{}) (*identity.Config, error) { if value, found := cfgmap["identity"]; found { subMap := value.(map[interface{}]interface{}) diff --git a/router/internal/edgerouter/config.go b/router/configedge.go similarity index 87% rename from router/internal/edgerouter/config.go rename to router/configedge.go index b6ed2364a..90b206207 100644 --- a/router/internal/edgerouter/config.go +++ b/router/configedge.go @@ -14,7 +14,7 @@ limitations under the License. */ -package edgerouter +package router import ( "fmt" @@ -30,7 +30,6 @@ import ( "github.com/openziti/transport/v2" "github.com/openziti/ziti/common" "github.com/openziti/ziti/common/pb/edge_ctrl_pb" - "github.com/openziti/ziti/router" "github.com/pkg/errors" "github.com/spf13/pflag" ) @@ -42,12 +41,9 @@ const ( DefaultSessionValidateChunkSize = 1000 DefaultSessionValidateMinInterval = "250ms" DefaultSessionValidateMaxInterval = "1500ms" - - FlagsCfgMapKey = "@flags" ) -type Config struct { - FilePath string +type EdgeConfig struct { Enabled bool ApiProxy ApiProxy EdgeListeners []*edge_ctrl_pb.Listener @@ -59,8 +55,7 @@ type Config struct { Tcfg transport.Configuration ForceExtendEnrollment bool - RouterConfig *router.Config - EnrollmentIdentityConfig *identity.Config + RouterConfig *Config Db string DbSaveInterval time.Duration @@ -81,13 +76,27 @@ type ApiProxy struct { Upstream string } -func NewConfig(routerConfig *router.Config) *Config { - return &Config{ +type Memory struct { + Path string `yaml:"path"` + Interval time.Duration `yaml:"intervalMs"` +} + +type Sans struct { + DnsAddresses []string `yaml:"dns" mapstructure:"dns"` + IpAddresses []string `yaml:"ip" mapstructure:"ip"` + IpAddressesParsed []net.IP + EmailAddresses []string `yaml:"email" mapstructure:"email"` + UriAddresses []string `yaml:"uri" mapstructure:"uri"` + UriAddressesParsed []*url.URL +} + +func NewEdgeConfig(routerConfig *Config) *EdgeConfig { + return &EdgeConfig{ RouterConfig: routerConfig, } } -func (config *Config) LoadConfigFromMap(configMap map[interface{}]interface{}) error { +func (config *EdgeConfig) LoadEdgeConfigFromMap(configMap map[interface{}]interface{}, loadIdentity bool) error { var err error config.Enabled = false @@ -99,17 +108,25 @@ func (config *Config) LoadConfigFromMap(configMap map[interface{}]interface{}) e } } - var edgeConfigMap map[interface{}]interface{} = nil + if err = config.loadCsr(configMap); err != nil { + return err + } + + var edgeConfigMap map[interface{}]interface{} if val, ok := configMap["edge"]; ok && val != nil { config.Enabled = true if edgeConfigMap, ok = val.(map[interface{}]interface{}); !ok { return fmt.Errorf("expected map as edge configuration") } + } else { + return nil } - if err := config.ensureIdentity(configMap); err != nil { - return err + if loadIdentity { + if err := config.ensureIdentity(configMap); err != nil { + return err + } } if val, found := edgeConfigMap["db"]; found { @@ -119,7 +136,7 @@ func (config *Config) LoadConfigFromMap(configMap map[interface{}]interface{}) e if config.Db == "" { config.Db = "./db.proto.gzip" - if value, found := configMap[router.PathMapKey]; found { + if value, found := configMap[PathMapKey]; found { configPath := value.(string) configPath = strings.TrimSpace(configPath) config.Db = configPath + ".proto.gzip" @@ -127,7 +144,7 @@ func (config *Config) LoadConfigFromMap(configMap map[interface{}]interface{}) e pfxlog.Logger().Warnf("the db property was not set, using default for cached data model: %s", config.Db) } - pfxlog.Logger().Infof("cached data model file set to: %s", config.Db) + pfxlog.Logger().Debugf("cached data model file set to: %s", config.Db) } if val, found := edgeConfigMap["dbSaveIntervalSeconds"]; found { @@ -177,11 +194,7 @@ func (config *Config) LoadConfigFromMap(configMap map[interface{}]interface{}) e return err } - if err = config.loadCsr(configMap); err != nil { - return err - } - - if err = config.loadListener(configMap); err != nil { + if err = config.loadEdgeListener(configMap); err != nil { return err } @@ -192,7 +205,7 @@ func (config *Config) LoadConfigFromMap(configMap map[interface{}]interface{}) e return nil } -func (config *Config) loadApiProxy(edgeConfigMap map[interface{}]interface{}) error { +func (config *EdgeConfig) loadApiProxy(edgeConfigMap map[interface{}]interface{}) error { config.ApiProxy = ApiProxy{} if value, found := edgeConfigMap["apiProxy"]; found { @@ -231,7 +244,7 @@ func (config *Config) loadApiProxy(edgeConfigMap map[interface{}]interface{}) er return nil } -func (config *Config) loadListener(rootConfigMap map[interface{}]interface{}) error { +func (config *EdgeConfig) loadEdgeListener(rootConfigMap map[interface{}]interface{}) error { subArray := rootConfigMap["listeners"] listeners, ok := subArray.([]interface{}) @@ -345,7 +358,7 @@ func parseEdgeListenerOptions(index int, address string, edgeListenerMap map[int } // loadCsr search for a root `csr` path or an `edge.csr` path for a CSR definition. The root path is preferred. -func (config *Config) loadCsr(rootConfigMap map[interface{}]interface{}) error { +func (config *EdgeConfig) loadCsr(rootConfigMap map[interface{}]interface{}) error { csrI, ok := rootConfigMap["csr"] csrPath := "csr" @@ -395,7 +408,7 @@ func (config *Config) loadCsr(rootConfigMap map[interface{}]interface{}) error { } // parseCsr parses the given map as a CSR definition. Error messages are based on the path provided. -func (config *Config) parseCsr(csrMap map[interface{}]interface{}, path string) (*Csr, error) { +func (config *EdgeConfig) parseCsr(csrMap map[interface{}]interface{}, path string) (*Csr, error) { targetCsr := &Csr{} if csrMap == nil { @@ -425,9 +438,9 @@ func (config *Config) parseCsr(csrMap map[interface{}]interface{}, path string) return targetCsr, nil } -func (config *Config) ensureIdentity(rootConfigMap map[interface{}]interface{}) error { +func (config *EdgeConfig) ensureIdentity(rootConfigMap map[interface{}]interface{}) error { if config.RouterConfig == nil { - config.RouterConfig = &router.Config{} + config.RouterConfig = &Config{} } //if we already have an Id (loaded by the fabric router) use that as is to avoid @@ -436,7 +449,7 @@ func (config *Config) ensureIdentity(rootConfigMap map[interface{}]interface{}) return nil } - idConfig, err := router.LoadIdentityConfigFromMap(rootConfigMap) + idConfig, err := LoadIdentityConfigFromMap(rootConfigMap) if err != nil { return err @@ -457,7 +470,7 @@ func (config *Config) ensureIdentity(rootConfigMap map[interface{}]interface{}) return nil } -func (config *Config) loadTransportConfig(rootConfigMap map[interface{}]interface{}) error { +func (config *EdgeConfig) loadTransportConfig(rootConfigMap map[interface{}]interface{}) error { if val, ok := rootConfigMap["transport"]; ok && val != nil { config.Tcfg = make(transport.Configuration) if tcfg, ok := val.(map[interface{}]interface{}); ok { @@ -471,20 +484,3 @@ func (config *Config) loadTransportConfig(rootConfigMap map[interface{}]interfac return nil } - -// LoadConfigFromMapForEnrollment loads a minimal subset of the router configuration to allow for enrollment. -// This process should be used to load edge enabled routers as well as non-edge routers. -func (config *Config) LoadConfigFromMapForEnrollment(cfgmap map[interface{}]interface{}) error { - var err error - config.EnrollmentIdentityConfig, err = router.LoadIdentityConfigFromMap(cfgmap) - - if err != nil { - return err - } - - if err := config.loadCsr(cfgmap); err != nil { - return fmt.Errorf("error loading csr: %w", err) - } - - return nil -} diff --git a/router/enroll/csr.go b/router/enroll/csr.go index 1cab8b7aa..c01831d30 100644 --- a/router/enroll/csr.go +++ b/router/enroll/csr.go @@ -22,7 +22,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" - "github.com/openziti/ziti/router/internal/edgerouter" + "github.com/openziti/ziti/router" ) type Csr struct { @@ -34,7 +34,7 @@ type Csr struct { SanUri []string } -func CreateCsr(key crypto.PrivateKey, algo x509.SignatureAlgorithm, subj *pkix.Name, sans *edgerouter.Sans) (string, error) { +func CreateCsr(key crypto.PrivateKey, algo x509.SignatureAlgorithm, subj *pkix.Name, sans *router.Sans) (string, error) { template := x509.CertificateRequest{ Subject: *subj, SignatureAlgorithm: algo, diff --git a/router/enroll/enroll.go b/router/enroll/enroll.go index 601f62449..dd500be79 100644 --- a/router/enroll/enroll.go +++ b/router/enroll/enroll.go @@ -36,7 +36,6 @@ import ( "github.com/openziti/sdk-golang/ziti" "github.com/openziti/sdk-golang/ziti/enroll" "github.com/openziti/ziti/router" - "github.com/openziti/ziti/router/internal/edgerouter" ) type apiPost struct { @@ -46,37 +45,18 @@ type apiPost struct { type Enroller interface { Enroll(jwt []byte, silent bool, engine string, keyAlg ziti.KeyAlgVar) error - LoadConfig(cfgmap map[interface{}]interface{}) error } type RestEnroller struct { - config *edgerouter.Config + fullConfig *router.Config + config *router.EdgeConfig } -func NewRestEnroller() Enroller { - return &RestEnroller{} -} - -func (re *RestEnroller) parseCfgMap(cfgmap map[interface{}]interface{}) (*edgerouter.Config, error) { - routerConfig := &router.Config{} - - edgeConfig := edgerouter.NewConfig(routerConfig) - if err := edgeConfig.LoadConfigFromMapForEnrollment(cfgmap); err != nil { - return nil, fmt.Errorf("could not load edge router config: %v", err) - } - - return edgeConfig, nil -} - -func (re *RestEnroller) LoadConfig(cfgmap map[interface{}]interface{}) error { - var err error - re.config, err = re.parseCfgMap(cfgmap) - - if err != nil { - return fmt.Errorf("error parsing configuration: %s", err) +func NewRestEnroller(config *router.Config) Enroller { + return &RestEnroller{ + fullConfig: config, + config: config.Edge, } - - return nil } func (re *RestEnroller) Enroll(jwtBuf []byte, silent bool, engine string, keyAlg ziti.KeyAlgVar) error { @@ -86,10 +66,9 @@ func (re *RestEnroller) Enroll(jwtBuf []byte, silent bool, engine string, keyAlg return errors.New("no configuration provided") } - identityConfig := re.config.EnrollmentIdentityConfig + identityConfig := re.fullConfig.IdConfig if re.config.RouterConfig.Id != nil { - identityConfig = re.config.RouterConfig.Id.GetConfig() log.Warnf("identity detected, note that any identity information will be overwritten when enrolling") } @@ -203,6 +182,10 @@ func (re *RestEnroller) Enroll(jwtBuf []byte, silent bool, engine string, keyAlg return fmt.Errorf("unable to write CA certs to [%s]: %s", identityConfig.CA, err) } + if err = re.fullConfig.SaveControllerEndpoints(ec.Controllers); err != nil { + return err + } + log.Info("registration complete") return nil } diff --git a/router/internal/apiproxy/proxy.go b/router/internal/apiproxy/proxy.go index ff84d770d..7444ea9b0 100644 --- a/router/internal/apiproxy/proxy.go +++ b/router/internal/apiproxy/proxy.go @@ -20,7 +20,7 @@ import ( "crypto/tls" "encoding/base64" "github.com/michaelquigley/pfxlog" - "github.com/openziti/ziti/router/internal/edgerouter" + "github.com/openziti/ziti/router" "net/http" "net/http/httputil" "net/url" @@ -39,7 +39,7 @@ type Config struct { DownstreamTlsConfig *tls.Config } -func Start(config *edgerouter.Config) { +func Start(config *router.EdgeConfig) { if !config.ApiProxy.Enabled { pfxlog.Logger().Debug("API Proxy disabled") diff --git a/router/internal/edgerouter/types.go b/router/internal/edgerouter/types.go deleted file mode 100644 index 65ccf20c8..000000000 --- a/router/internal/edgerouter/types.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package edgerouter - -import ( - "net" - "net/url" - "time" -) - -type Memory struct { - Path string `yaml:"path"` - Interval time.Duration `yaml:"intervalMs"` -} - -type Sans struct { - DnsAddresses []string `yaml:"dns" mapstructure:"dns"` - IpAddresses []string `yaml:"ip" mapstructure:"ip"` - IpAddressesParsed []net.IP - EmailAddresses []string `yaml:"email" mapstructure:"email"` - UriAddresses []string `yaml:"uri" mapstructure:"uri"` - UriAddressesParsed []*url.URL -} diff --git a/router/router.go b/router/router.go index 9454b0b17..39af366bb 100644 --- a/router/router.go +++ b/router/router.go @@ -27,9 +27,10 @@ import ( "github.com/openziti/ziti/router/state" "io/fs" "os" - "path" + "path/filepath" "plugin" "runtime/debug" + "strings" "sync/atomic" "time" @@ -267,8 +268,8 @@ func (self *Router) GetConfig() *Config { } func (self *Router) Start() error { - if err := os.MkdirAll(self.config.Ctrl.DataDir, 0700); err != nil { - logrus.WithField("dir", self.config.Ctrl.DataDir).WithError(err).Error("failed to initialize data directory") + if err := os.MkdirAll(filepath.Dir(self.config.Ctrl.EndpointsFile), 0700); err != nil { + logrus.WithField("dir", filepath.Dir(self.config.Ctrl.EndpointsFile)).WithError(err).Error("failed to initialize directory for endpoints file") return err } @@ -741,11 +742,11 @@ func (self *Router) RegisterXWebHandlerFactory(x xweb.ApiHandlerFactory) error { func (self *Router) getInitialCtrlEndpoints() ([]string, error) { log := pfxlog.Logger() - if self.config.Ctrl.DataDir == "" { - return nil, errors.New("ctrl DataDir not configured") + if self.config.Ctrl.EndpointsFile == "" { + return nil, errors.New("ctrl endpointsFile not configured") } - endpointsFile := path.Join(self.config.Ctrl.DataDir, "endpoints") + endpointsFile := self.config.Ctrl.EndpointsFile var endpoints []string @@ -758,12 +759,20 @@ func (self *Router) getInitialCtrlEndpoints() ([]string, error) { if err != nil { log.WithError(err).Error("unable to read endpoints file, falling back to initial endpoints from config") } else { - endpointCfg := &endpointConfig{} + endpointCfg := map[string]any{} if err = yaml.Unmarshal(b, endpointCfg); err != nil { log.WithError(err).Error("unable to unmarshall endpoints file, falling back to initial endpoints from config") } else { - endpoints = endpointCfg.Endpoints + for k, v := range endpointCfg { + if strings.EqualFold("endpoints", k) { + keys := v.([]any) + for _, key := range keys { + endpoints = append(endpoints, key.(string)) + } + break + } + } if len(endpoints) == 0 { log.Info("empty endpoint list in endpoints file, falling back to initial endpoints from config") } @@ -782,19 +791,11 @@ func (self *Router) getInitialCtrlEndpoints() ([]string, error) { } func (self *Router) UpdateCtrlEndpoints(endpoints []string) { - log := pfxlog.Logger().WithField("endpoints", endpoints).WithField("filepath", self.config.Ctrl.DataDir) + log := pfxlog.Logger().WithField("endpoints", endpoints).WithField("filepath", self.config.Ctrl.EndpointsFile) if changed := self.ctrls.UpdateControllerEndpoints(endpoints); changed { log.Info("Attempting to save file") - endpointsFile := path.Join(self.config.Ctrl.DataDir, "endpoints") - - configData := map[string]interface{}{ - "Endpoints": endpoints, - } - - if data, err := yaml.Marshal(configData); err != nil { - log.WithError(err).Error("unable to marshal updated controller endpoints to yaml") - } else if err = os.WriteFile(endpointsFile, data, 0600); err != nil { - log.WithError(err).Error("unable to write updated controller endpoints to file") + if err := self.config.SaveControllerEndpoints(endpoints); err != nil { + log.WithError(err).Error("unable to save updated endpoints file") } } } @@ -834,7 +835,7 @@ func (self *controllerPinger) PingContext(context.Context) error { } type endpointConfig struct { - Endpoints []string `yaml:"Endpoints"` + Endpoints []string `yaml:"endpoints"` } type linkConnDetail struct { diff --git a/router/router_test.go b/router/router_test.go index 215ffac73..4c2e75b41 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "os" "path" + "path/filepath" "testing" "time" @@ -27,7 +28,7 @@ func Test_initializeCtrlEndpoints_ErrorsWithoutDataDir(t *testing.T) { } _, err = r.getInitialCtrlEndpoints() assert.Error(t, err) - assert.ErrorContains(t, err, "ctrl DataDir not configured") + assert.ErrorContains(t, err, "ctrl endpointsFile not configured") } func Test_initializeCtrlEndpoints(t *testing.T) { @@ -48,12 +49,12 @@ func Test_initializeCtrlEndpoints(t *testing.T) { LocalBinding string DefaultRequestTimeout time.Duration Options *channel.Options - DataDir string + EndpointsFile string Heartbeats env.HeartbeatOptions StartupTimeout time.Duration RateLimit command.AdaptiveRateLimiterConfig }{ - DataDir: tmpDir, + EndpointsFile: filepath.Join(tmpDir, "endpoints"), InitialEndpoints: []*UpdatableAddress{NewUpdatableAddress(addr)}, }, }, @@ -90,12 +91,12 @@ func Test_updateCtrlEndpoints(t *testing.T) { LocalBinding string DefaultRequestTimeout time.Duration Options *channel.Options - DataDir string + EndpointsFile string Heartbeats env.HeartbeatOptions StartupTimeout time.Duration RateLimit command.AdaptiveRateLimiterConfig }{ - DataDir: tmpDir, + EndpointsFile: filepath.Join(tmpDir, "endpoints"), InitialEndpoints: []*UpdatableAddress{NewUpdatableAddress(addr), NewUpdatableAddress(addr2)}, }, }, diff --git a/router/xgress_edge/certchecker.go b/router/xgress_edge/certchecker.go index 5f3bb0753..e02e048b2 100644 --- a/router/xgress_edge/certchecker.go +++ b/router/xgress_edge/certchecker.go @@ -21,12 +21,12 @@ import ( "fmt" "github.com/michaelquigley/pfxlog" "github.com/openziti/channel/v3" + "github.com/openziti/identity" "github.com/openziti/ziti/common/pb/edge_ctrl_pb" "github.com/openziti/ziti/controller/env" + "github.com/openziti/ziti/router" "github.com/openziti/ziti/router/enroll" - "github.com/openziti/ziti/router/internal/edgerouter" routerEnv "github.com/openziti/ziti/router/env" - "github.com/openziti/identity" "github.com/pkg/errors" "google.golang.org/protobuf/proto" "sync/atomic" @@ -46,7 +46,7 @@ type CertExpirationChecker struct { id *identity.TokenId closeNotify <-chan struct{} ctrls routerEnv.NetworkControllers - edgeConfig *edgerouter.Config + edgeConfig *router.EdgeConfig certsUpdated chan struct{} isRunning atomic.Bool @@ -58,7 +58,7 @@ type CertExpirationChecker struct { extender CertExtender } -func NewCertExpirationChecker(id *identity.TokenId, edgeConfig *edgerouter.Config, ctrls routerEnv.NetworkControllers, closeNotify <-chan struct{}) *CertExpirationChecker { +func NewCertExpirationChecker(id *identity.TokenId, edgeConfig *router.EdgeConfig, ctrls routerEnv.NetworkControllers, closeNotify <-chan struct{}) *CertExpirationChecker { ret := &CertExpirationChecker{ id: id, closeNotify: closeNotify, diff --git a/router/xgress_edge/certchecker_test.go b/router/xgress_edge/certchecker_test.go index 9bb3129ed..53300cdfb 100644 --- a/router/xgress_edge/certchecker_test.go +++ b/router/xgress_edge/certchecker_test.go @@ -13,8 +13,8 @@ import ( "github.com/openziti/identity" "github.com/openziti/transport/v2" "github.com/openziti/ziti/common/eid" + "github.com/openziti/ziti/router" "github.com/openziti/ziti/router/env" - "github.com/openziti/ziti/router/internal/edgerouter" "github.com/pkg/errors" "github.com/stretchr/testify/require" "math/big" @@ -536,7 +536,7 @@ func newCertChecker() (*CertExpirationChecker, func()) { } time.Sleep(10 * time.Millisecond) } - return NewCertExpirationChecker(id, &edgerouter.Config{}, ctrls, closeNotify), func() { close(closeNotify) } + return NewCertExpirationChecker(id, &router.EdgeConfig{}, ctrls, closeNotify), func() { close(closeNotify) } } type simpleTestUnderlay struct{} diff --git a/router/xgress_edge/factory.go b/router/xgress_edge/factory.go index 7d02a4698..381df210d 100644 --- a/router/xgress_edge/factory.go +++ b/router/xgress_edge/factory.go @@ -21,6 +21,7 @@ import ( "github.com/michaelquigley/pfxlog" "github.com/openziti/channel/v3" "github.com/openziti/foundation/v2/concurrenz" + "github.com/openziti/foundation/v2/stringz" "github.com/openziti/foundation/v2/versions" "github.com/openziti/metrics" "github.com/openziti/sdk-golang/ziti/edge" @@ -31,7 +32,6 @@ import ( "github.com/openziti/ziti/router/env" "github.com/openziti/ziti/router/handler_edge_ctrl" "github.com/openziti/ziti/router/internal/apiproxy" - "github.com/openziti/ziti/router/internal/edgerouter" "github.com/openziti/ziti/router/state" "github.com/openziti/ziti/router/xgress" "github.com/pkg/errors" @@ -47,7 +47,7 @@ type Factory struct { ctrls env.NetworkControllers enabled bool routerConfig *router.Config - edgeRouterConfig *edgerouter.Config + edgeRouterConfig *router.EdgeConfig hostedServices *hostedServiceRegistry stateManager state.Manager versionProvider versions.VersionProvider @@ -119,23 +119,25 @@ func (factory *Factory) Run(env env.RouterEnv) error { } func (factory *Factory) LoadConfig(configMap map[interface{}]interface{}) error { - _, factory.enabled = configMap["edge"] + factory.enabled = factory.routerConfig.Edge.Enabled if !factory.enabled { return nil } - var err error - edgeConfig := edgerouter.NewConfig(factory.routerConfig) - if err = edgeConfig.LoadConfigFromMap(configMap); err != nil { - return err - } + edgeConfig := factory.routerConfig.Edge if edgeConfig.Tcfg == nil { edgeConfig.Tcfg = make(transport.Configuration) } - edgeConfig.Tcfg["protocol"] = append(edgeConfig.Tcfg.Protocols(), "ziti-edge", "") - factory.edgeRouterConfig = edgeConfig + if !stringz.Contains(edgeConfig.Tcfg.Protocols(), "ziti-edge") { + edgeConfig.Tcfg[transport.KeyProtocol] = append(edgeConfig.Tcfg.Protocols(), "ziti-edge") + } + if !stringz.Contains(edgeConfig.Tcfg.Protocols(), "") { + edgeConfig.Tcfg[transport.KeyProtocol] = append(edgeConfig.Tcfg.Protocols(), "") + } + + factory.edgeRouterConfig = factory.routerConfig.Edge factory.stateManager.LoadRouterModel(factory.edgeRouterConfig.Db) factory.env.MarkRouterDataModelRequired() diff --git a/tests/context.go b/tests/context.go index 6a4872c3f..13196168c 100644 --- a/tests/context.go +++ b/tests/context.go @@ -467,11 +467,10 @@ func (ctx *TestContext) requireEnrollEdgeRouter(tunneler bool, routerId string) if tunneler { configFile = TunnelerEdgeRouterConfFile } - configMap, err := router.LoadConfigMap(configFile) + routerConfig, err := router.LoadConfigWithOptions(configFile, false) ctx.Req.NoError(err) - enroller := enroll.NewRestEnroller() - ctx.Req.NoError(enroller.LoadConfig(configMap)) + enroller := enroll.NewRestEnroller(routerConfig) var keyAlg ziti.KeyAlgVar _ = keyAlg.Set("RSA") ctx.Req.NoError(enroller.Enroll([]byte(jwt), true, "", keyAlg)) @@ -489,11 +488,10 @@ func (ctx *TestContext) createAndEnrollTransitRouter() *transitRouter { ctx.transitRouterEntity = ctx.AdminManagementSession.requireNewTransitRouter() jwt := ctx.AdminManagementSession.getTransitRouterJwt(ctx.transitRouterEntity.id) - configMap, err := router.LoadConfigMap(TransitRouterConfFile) + routerConfig, err := router.LoadConfigWithOptions(TransitRouterConfFile, false) ctx.Req.NoError(err) - enroller := enroll.NewRestEnroller() - ctx.Req.NoError(enroller.LoadConfig(configMap)) + enroller := enroll.NewRestEnroller(routerConfig) var keyAlg ziti.KeyAlgVar _ = keyAlg.Set("RSA") ctx.Req.NoError(enroller.Enroll([]byte(jwt), true, "", keyAlg)) diff --git a/tests/endpoints b/tests/endpoints index c5faf197e..bb3c7f4b1 100644 --- a/tests/endpoints +++ b/tests/endpoints @@ -1 +1 @@ -Endpoints: ['tls:127.0.0.1:6262'] +endpoints: [] diff --git a/ziti/router/enrollgw.go b/ziti/router/enrollgw.go index 4ed5c3a69..f8850345e 100644 --- a/ziti/router/enrollgw.go +++ b/ziti/router/enrollgw.go @@ -18,9 +18,9 @@ package router import ( "github.com/michaelquigley/pfxlog" - "github.com/openziti/ziti/router/enroll" - "github.com/openziti/ziti/router" "github.com/openziti/sdk-golang/ziti" + "github.com/openziti/ziti/router" + "github.com/openziti/ziti/router/enroll" "github.com/spf13/cobra" "os" ) @@ -49,15 +49,10 @@ func NewEnrollGwCmd() *cobra.Command { func enrollGw(cmd *cobra.Command, args []string) { log := pfxlog.Logger() - if cfgmap, err := router.LoadConfigMap(args[0]); err == nil { - router.SetConfigMapFlags(cfgmap, getFlags(cmd)) + if cfg, err := router.LoadConfigWithOptions(args[0], false); err == nil { + cfg.SetFlags(getFlags(cmd)) - enroller := enroll.NewRestEnroller() - err := enroller.LoadConfig(cfgmap) - - if err != nil { - log.Panicf("could not load config: %s", err) - } + enroller := enroll.NewRestEnroller(cfg) jwtBuf, err := os.ReadFile(*jwtPath) if err != nil {