diff --git a/cmd/microcloud/add.go b/cmd/microcloud/add.go index 3424b5527..2c3dbcd09 100644 --- a/cmd/microcloud/add.go +++ b/cmd/microcloud/add.go @@ -73,11 +73,15 @@ func (c *cmdAdd) Run(cmd *cobra.Command, args []string) error { cfg.name = status.Name cfg.address = status.Address.Addr().String() - err = cfg.askAddress("") + microcloudInternalNet, err := cfg.askAddress("") if err != nil { return err } + addingSystem := cfg.systems[cfg.name] + addingSystem.MicroCloudInternalNetwork = microcloudInternalNet + cfg.systems[cfg.name] = addingSystem + installedServices := []types.ServiceType{types.MicroCloud, types.LXD} optionalServices := map[types.ServiceType]string{ types.MicroCeph: api.MicroCephDir, diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index 86f0ee42c..98dcb8dbe 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -200,21 +200,21 @@ func (c *initConfig) askMissingServices(services []types.ServiceType, stateDirs return services, nil } -func (c *initConfig) askAddress(filterAddress string) error { +func (c *initConfig) askAddress(filterAddress string) (*Network, error) { info, err := multicast.GetNetworkInfo() if err != nil { - return fmt.Errorf("Failed to find network interfaces: %w", err) + return nil, fmt.Errorf("Failed to find network interfaces: %w", err) } listenAddr := c.address if listenAddr == "" { if len(info) == 0 { - return fmt.Errorf("Found no valid network interfaces") + return nil, fmt.Errorf("Found no valid network interfaces") } filterIp := net.ParseIP(filterAddress) if filterAddress != "" && filterIp == nil { - return fmt.Errorf("Invalid filter address %q", filterAddress) + return nil, fmt.Errorf("Invalid filter address %q", filterAddress) } listenAddr = info[0].Address @@ -246,7 +246,7 @@ func (c *initConfig) askAddress(filterAddress string) error { return nil }) if err != nil { - return err + return nil, err } } else { fmt.Println(tui.SummarizeResult("Using address %s for MicroCloud", listenAddr)) @@ -264,14 +264,19 @@ func (c *initConfig) askAddress(filterAddress string) error { } if subnet == nil { - return fmt.Errorf("Cloud not find valid subnet for address %q", listenAddr) + return nil, fmt.Errorf("Cloud not find valid subnet for address %q", listenAddr) } c.address = listenAddr c.lookupIface = iface c.lookupSubnet = subnet - return nil + _, cidrMicroCloudSubnet, err := net.ParseCIDR(c.lookupSubnet.String()) + if err != nil { + return nil, fmt.Errorf("Failed to parse MicroCloud internal CIDR subnet: %w", err) + } + + return &Network{Interface: *iface, Subnet: cidrMicroCloudSubnet, IP: net.IP(listenAddr)}, nil } func (c *initConfig) askDisks(sh *service.Handler) error { @@ -1149,7 +1154,7 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { } } - var ovnUnderlaySelectedIPs map[string]string + var ovnUnderlaySelectedNets map[string]*Network ovnUnderlayData := [][]string{} for peer, system := range c.systems { // skip any systems that have already been clustered, but are available for other configuration. @@ -1175,7 +1180,6 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { if wantsDedicatedUnderlay { header = []string{"LOCATION", "IFACE", "TYPE", "IP ADDRESS (CIDR)"} - ovnUnderlaySelectedIPs = map[string]string{} err = c.askRetry("Retry selecting underlay network interfaces?", func() error { table := tui.NewSelectableTable(header, ovnUnderlayData) answers, err := table.Render(context.Background(), c.asker, "Select exactly one network interface from each cluster member:") @@ -1183,20 +1187,22 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { return err } - ovnUnderlaySelectedIPs = map[string]string{} + ovnUnderlaySelectedNets = make(map[string]*Network) for _, answer := range answers { target := answer["LOCATION"] ipAddr := answer["IP ADDRESS (CIDR)"] - if ovnUnderlaySelectedIPs[target] != "" { + ifaceName := answer["IFACE"] + + if ovnUnderlaySelectedNets[target] != nil { return fmt.Errorf("Failed to configure OVN underlay traffic: Selected more than one interface for target %q", target) } - ip, _, err := net.ParseCIDR(ipAddr) + ip, ipNet, err := net.ParseCIDR(ipAddr) if err != nil { return err } - ovnUnderlaySelectedIPs[target] = ip.String() + ovnUnderlaySelectedNets[target] = &Network{Interface: net.Interface{Name: ifaceName}, IP: ip, Subnet: ipNet} } return nil @@ -1207,11 +1213,11 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { } } - if len(ovnUnderlaySelectedIPs) > 0 { + if len(ovnUnderlaySelectedNets) > 0 { for peer := range askSystems { - underlayIP, ok := ovnUnderlaySelectedIPs[peer] + underlayNetwork, ok := ovnUnderlaySelectedNets[peer] if ok { - fmt.Printf(" Using %q for OVN underlay traffic on %q\n", underlayIP, peer) + fmt.Printf(" Using %q for OVN underlay traffic on %q\n", underlayNetwork.IP.String(), peer) } } @@ -1248,10 +1254,10 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { system.Networks = append(system.Networks, finalConfigs...) } - if ovnUnderlaySelectedIPs != nil { - ovnUnderlayIpAddr, ok := ovnUnderlaySelectedIPs[peer] + if ovnUnderlaySelectedNets != nil { + ovnUnderlayNet, ok := ovnUnderlaySelectedNets[peer] if ok { - system.OVNGeneveAddr = ovnUnderlayIpAddr + system.OVNGeneveNetwork = ovnUnderlayNet } } @@ -1409,15 +1415,29 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error { return err } + internalCephNetworkInterface, err := lxd.FindInterfaceForSubnet(internalCephSubnet) + if err != nil { + return fmt.Errorf("Failed to find interface for subnet %q: %w", internalCephSubnet, err) + } + if internalCephSubnet != microCloudInternalNetworkAddrCIDR { err = c.validateCephInterfacesForSubnet(lxd, availableCephNetworkInterfaces, internalCephSubnet) if err != nil { return err } - bootstrapSystem := c.systems[sh.Name] - bootstrapSystem.MicroCephInternalNetworkSubnet = internalCephSubnet - c.systems[sh.Name] = bootstrapSystem + internalCephIP, internalCephNet, err := net.ParseCIDR(internalCephSubnet) + if err != nil { + return fmt.Errorf("Failed to parse the internal Ceph network: %w", err) + } + + bootstrapSystem := c.systems[c.name] + bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: *internalCephNetworkInterface, Subnet: internalCephNet, IP: internalCephIP} + c.systems[c.name] = bootstrapSystem + } else { + bootstrapSystem := c.systems[c.name] + bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: bootstrapSystem.MicroCloudInternalNetwork.Interface, Subnet: bootstrapSystem.MicroCloudInternalNetwork.Subnet, IP: bootstrapSystem.MicroCloudInternalNetwork.IP} + c.systems[c.name] = bootstrapSystem } publicCephSubnet, err := c.asker.AskString("What subnet (either IPv4 or IPv6 CIDR notation) would you like your Ceph public traffic on?", internalCephSubnet, validate.IsNetwork) @@ -1425,6 +1445,11 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error { return err } + publicCephNetworkInterface, err := lxd.FindInterfaceForSubnet(publicCephSubnet) + if err != nil { + return fmt.Errorf("Failed to find interface for subnet %q: %w", publicCephSubnet, err) + } + if publicCephSubnet != internalCephSubnet { err = c.validateCephInterfacesForSubnet(lxd, availableCephNetworkInterfaces, publicCephSubnet) if err != nil { @@ -1433,17 +1458,31 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error { } if publicCephSubnet != microCloudInternalNetworkAddrCIDR { + publicCephIP, publicCephNet, err := net.ParseCIDR(publicCephSubnet) + if err != nil { + return fmt.Errorf("Failed to parse the public Ceph network: %w", err) + } + bootstrapSystem := c.systems[sh.Name] - bootstrapSystem.MicroCephPublicNetworkSubnet = publicCephSubnet + bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: *publicCephNetworkInterface, Subnet: publicCephNet, IP: publicCephIP} c.systems[sh.Name] = bootstrapSystem // This is to avoid the situation where the internal network for Ceph has been skipped, but the public network has been set. // Ceph will automatically set the internal network to the public Ceph network if the internal network is not set, which is not what we want. // Instead, we still want to keep the internal Ceph network to use the MicroCloud internal network as a default. if internalCephSubnet == microCloudInternalNetworkAddrCIDR { - bootstrapSystem.MicroCephInternalNetworkSubnet = microCloudInternalNetworkAddrCIDR + microcloudInternalIP, microcloudNet, err := net.ParseCIDR(microCloudInternalNetworkAddrCIDR) + if err != nil { + return fmt.Errorf("Failed to parse the internal MicroCloud network: %w", err) + } + + bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: bootstrapSystem.MicroCloudInternalNetwork.Interface, Subnet: microcloudNet, IP: microcloudInternalIP} c.systems[sh.Name] = bootstrapSystem } + } else { + bootstrapSystem := c.systems[sh.Name] + bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: bootstrapSystem.MicroCloudInternalNetwork.Interface, Subnet: bootstrapSystem.MicroCloudInternalNetwork.Subnet, IP: bootstrapSystem.MicroCloudInternalNetwork.IP} + c.systems[sh.Name] = bootstrapSystem } return nil diff --git a/cmd/microcloud/join.go b/cmd/microcloud/join.go index 6a07b6e2f..f69cfd5c1 100644 --- a/cmd/microcloud/join.go +++ b/cmd/microcloud/join.go @@ -65,16 +65,20 @@ func (c *cmdJoin) Run(cmd *cobra.Command, args []string) error { cfg.sessionTimeout = time.Duration(c.flagSessionTimeout) * time.Second } - err = cfg.askAddress(c.flagInitiatorAddress) + cfg.name, err = os.Hostname() if err != nil { - return err + return fmt.Errorf("Failed to retrieve system hostname: %w", err) } - cfg.name, err = os.Hostname() + microcloudInternalNet, err := cfg.askAddress(c.flagInitiatorAddress) if err != nil { - return fmt.Errorf("Failed to retrieve system hostname: %w", err) + return err } + joiningSystem := cfg.systems[cfg.name] + joiningSystem.MicroCloudInternalNetwork = microcloudInternalNet + cfg.systems[cfg.name] = joiningSystem + installedServices := []types.ServiceType{types.MicroCloud, types.LXD} optionalServices := map[types.ServiceType]string{ types.MicroCeph: api.MicroCephDir, diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index 456bdcdf6..f2846c093 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -53,19 +53,24 @@ type InitSystem struct { AvailableDisks []lxdAPI.ResourcesStorageDisk // MicroCephDisks contains the disks intended to be passed to MicroCeph. MicroCephDisks []cephTypes.DisksPost - // MicroCephPublicNetworkSubnet is an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph public network. - MicroCephPublicNetworkSubnet string - // MicroCephClusterNetworkSubnet is an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph cluster network. - MicroCephInternalNetworkSubnet string + // MicroCephPublicNetwork contains an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph public network and + // the network interface name to use for the Ceph public network and its IP address within the subnet. + MicroCephPublicNetwork *Network + // MicroCephInternalNetwork contains an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph cluster network and + // the network interface name to use for the Ceph cluster network and its IP address within the subnet. + MicroCephInternalNetwork *Network + // MicroCloudInternalNetwork contains the network configuration for the MicroCloud internal network. + MicroCloudInternalNetwork *Network // TargetNetworks contains the network configuration for the target system. TargetNetworks []lxdAPI.NetworksPost // TargetStoragePools contains the storage pool configuration for the target system. TargetStoragePools []lxdAPI.StoragePoolsPost // Networks is the cluster-wide network configuration. Networks []lxdAPI.NetworksPost - // OVNGeneveAddr represents an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system. + // OVNGeneveNetwork contains an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system. // If left empty, the system will choose to route the Geneve traffic through the management network. - OVNGeneveAddr string + // It also contains the network interface name to use for the OVN Geneve tunnel and the network subnet. + OVNGeneveNetwork *Network // StoragePools is the cluster-wide storage pool configuration. StoragePools []lxdAPI.StoragePoolsPost // StorageVolumes is the cluster-wide storage volume configuration. @@ -170,14 +175,14 @@ func (c *initConfig) RunInteractive(cmd *cobra.Command, args []string) error { return err } - err = c.askAddress("") + c.name, err = os.Hostname() if err != nil { - return err + return fmt.Errorf("Failed to retrieve system hostname: %w", err) } - c.name, err = os.Hostname() + microcloudInternalNet, err := c.askAddress("") if err != nil { - return fmt.Errorf("Failed to retrieve system hostname: %w", err) + return err } c.systems[c.name] = InitSystem{ @@ -185,6 +190,7 @@ func (c *initConfig) RunInteractive(cmd *cobra.Command, args []string) error { Name: c.name, Address: c.address, }, + MicroCloudInternalNetwork: microcloudInternalNet, } installedServices := []types.ServiceType{types.MicroCloud, types.LXD} @@ -388,9 +394,9 @@ func (c *initConfig) addPeers(sh *service.Handler) (revert.Hook, error) { CephConfig: info.MicroCephDisks, } - if info.OVNGeneveAddr != "" { + if info.OVNGeneveNetwork != nil { p := joinConfig[peer] - p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveAddr} + p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveNetwork.IP.String()} joinConfig[peer] = p } } @@ -694,12 +700,12 @@ func (c *initConfig) setupCluster(s *service.Handler) error { if s.Type() == types.MicroCeph { microCephBootstrapConf := make(map[string]string) - if bootstrapSystem.MicroCephInternalNetworkSubnet != "" { - microCephBootstrapConf["ClusterNet"] = bootstrapSystem.MicroCephInternalNetworkSubnet + if bootstrapSystem.MicroCephInternalNetwork != nil { + microCephBootstrapConf["ClusterNet"] = bootstrapSystem.MicroCephInternalNetwork.Subnet.String() } - if bootstrapSystem.MicroCephPublicNetworkSubnet != "" { - microCephBootstrapConf["PublicNet"] = bootstrapSystem.MicroCephPublicNetworkSubnet + if bootstrapSystem.MicroCephPublicNetwork != nil { + microCephBootstrapConf["PublicNet"] = bootstrapSystem.MicroCephPublicNetwork.Subnet.String() } if len(microCephBootstrapConf) > 0 { @@ -709,8 +715,8 @@ func (c *initConfig) setupCluster(s *service.Handler) error { if s.Type() == types.MicroOVN { microOvnBootstrapConf := make(map[string]string) - if bootstrapSystem.OVNGeneveAddr != "" { - microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveAddr + if bootstrapSystem.OVNGeneveNetwork != nil { + microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveNetwork.IP.String() } if len(microOvnBootstrapConf) > 0 { diff --git a/cmd/microcloud/preseed.go b/cmd/microcloud/preseed.go index 2cee2af08..f22dfcc2b 100644 --- a/cmd/microcloud/preseed.go +++ b/cmd/microcloud/preseed.go @@ -794,6 +794,8 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map return nil, fmt.Errorf("Failed to parse supplied underlay IP %q", sys.UnderlayIP) } + ovnUnderlayIfaceByPeer := make(map[string]string) + ovnUnderlaySubnetByPeer := make(map[string]*net.IPNet) for _, iface := range addressedInterfaces[sys.Name] { for _, cidr := range iface.Addresses { _, subnet, err := net.ParseCIDR(cidr) @@ -803,6 +805,8 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map if subnet.Contains(underlayIP) { assignedSystems[sys.Name] = true + ovnUnderlayIfaceByPeer[sys.Name] = iface.Network.Name + ovnUnderlaySubnetByPeer[sys.Name] = subnet break } } @@ -812,8 +816,18 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map return nil, fmt.Errorf("No available interface found for OVN underlay IP %q", sys.UnderlayIP) } + ifaceName, ok := ovnUnderlayIfaceByPeer[sys.Name] + if !ok { + return nil, fmt.Errorf("Failed to find OVN underlay interface name for system %q", sys.Name) + } + + subnet, ok := ovnUnderlaySubnetByPeer[sys.Name] + if !ok { + return nil, fmt.Errorf("Failed to find OVN underlay subnet for system %q", sys.Name) + } + system := c.systems[sys.Name] - system.OVNGeneveAddr = sys.UnderlayIP + system.OVNGeneveNetwork = &Network{Interface: net.Interface{Name: ifaceName}, Subnet: subnet, IP: underlayIP} c.systems[sys.Name] = system } } @@ -1101,8 +1115,22 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map return nil, err } + iface, err := lxd.FindInterfaceForSubnet(internalCephNetwork) + if err != nil { + return nil, err + } + + _, internalCephSubnet, err := net.ParseCIDR(internalCephNetwork) + if err != nil { + return nil, fmt.Errorf("Invalid CIDR for internal Ceph subnet: %w", err) + } + + bootstrapSystem := c.systems[s.Name] + bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: net.Interface{Name: iface.Name}, Subnet: internalCephSubnet, IP: nil} + c.systems[s.Name] = bootstrapSystem + } else { bootstrapSystem := c.systems[s.Name] - bootstrapSystem.MicroCephInternalNetworkSubnet = internalCephNetwork + bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: bootstrapSystem.MicroCloudInternalNetwork.Interface, Subnet: bootstrapSystem.MicroCloudInternalNetwork.Subnet, IP: bootstrapSystem.MicroCloudInternalNetwork.IP} c.systems[s.Name] = bootstrapSystem } @@ -1119,8 +1147,22 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map return nil, err } + iface, err := lxd.FindInterfaceForSubnet(publicCephNetwork) + if err != nil { + return nil, err + } + + _, publicCephSubnet, err := net.ParseCIDR(publicCephNetwork) + if err != nil { + return nil, fmt.Errorf("Invalid CIDR for public Ceph subnet: %w", err) + } + + bootstrapSystem := c.systems[s.Name] + bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: *iface, Subnet: publicCephSubnet, IP: nil} + c.systems[s.Name] = bootstrapSystem + } else { bootstrapSystem := c.systems[s.Name] - bootstrapSystem.MicroCephPublicNetworkSubnet = publicCephNetwork + bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: bootstrapSystem.MicroCloudInternalNetwork.Interface, Subnet: bootstrapSystem.MicroCloudInternalNetwork.Subnet, IP: bootstrapSystem.MicroCloudInternalNetwork.IP} c.systems[s.Name] = bootstrapSystem } } else { diff --git a/cmd/microcloud/services.go b/cmd/microcloud/services.go index 0afe9a312..b1fe7cfbf 100644 --- a/cmd/microcloud/services.go +++ b/cmd/microcloud/services.go @@ -242,11 +242,15 @@ func (c *cmdServiceAdd) Run(cmd *cobra.Command, args []string) error { cfg.address = status.Address.Addr().String() // enable auto setup to skip lookup related questions. cfg.autoSetup = true - err = cfg.askAddress("") + microcloudInternalNet, err := cfg.askAddress("") if err != nil { return err } + addingSystem := cfg.systems[cfg.name] + addingSystem.MicroCloudInternalNetwork = microcloudInternalNet + cfg.systems[cfg.name] = addingSystem + cfg.autoSetup = false installedServices := []types.ServiceType{types.MicroCloud, types.LXD} optionalServices := map[types.ServiceType]string{