Skip to content

Commit

Permalink
cmd/microcloud: refactor microcloud with new Network struct
Browse files Browse the repository at this point in the history 
* This introduce the `OVNGeneveNetwork`, `MicroCephPublicNetwork`, `MicroCephInternalNetwork` and `MicroCloudInternalNetwork` instead of `OVNGeneveAddr`, `MicroCephPublicNetworkSubnet`, `MicroCephInternalNetworkSubnet` and MicroCloud internal network information.

Signed-off-by: Gabriel Mougard <[email protected]>
  • Loading branch information
@gabrielmougard
gabrielmougard committed Feb 10, 2025
1 parent 21d3c7b commit 64fdc39
Show file tree
Hide file tree
Showing 6 changed files with 150 additions and 51 deletions.
6 changes: 5 additions & 1 deletion cmd/microcloud/add.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,15 @@ func (c *cmdAdd) Run(cmd *cobra.Command, args []string) error {

cfg.name = status.Name
cfg.address = status.Address.Addr().String()
err = cfg.askAddress("")
microcloudInternalNet, err := cfg.askAddress("")
if err != nil {
return err
}

addingSystem := cfg.systems[cfg.name]
addingSystem.MicroCloudInternalNetwork = microcloudInternalNet
cfg.systems[cfg.name] = addingSystem

installedServices := []types.ServiceType{types.MicroCloud, types.LXD}
optionalServices := map[types.ServiceType]string{
types.MicroCeph: api.MicroCephDir,
Expand Down
87 changes: 63 additions & 24 deletions cmd/microcloud/ask.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,21 +200,21 @@ func (c *initConfig) askMissingServices(services []types.ServiceType, stateDirs
return services, nil
}

func (c *initConfig) askAddress(filterAddress string) error {
func (c *initConfig) askAddress(filterAddress string) (*Network, error) {
info, err := multicast.GetNetworkInfo()
if err != nil {
return fmt.Errorf("Failed to find network interfaces: %w", err)
return nil, fmt.Errorf("Failed to find network interfaces: %w", err)
}

listenAddr := c.address
if listenAddr == "" {
if len(info) == 0 {
return fmt.Errorf("Found no valid network interfaces")
return nil, fmt.Errorf("Found no valid network interfaces")
}

filterIp := net.ParseIP(filterAddress)
if filterAddress != "" && filterIp == nil {
return fmt.Errorf("Invalid filter address %q", filterAddress)
return nil, fmt.Errorf("Invalid filter address %q", filterAddress)
}

listenAddr = info[0].Address
Expand Down Expand Up @@ -246,7 +246,7 @@ func (c *initConfig) askAddress(filterAddress string) error {
return nil
})
if err != nil {
return err
return nil, err
}
} else {
fmt.Println(tui.SummarizeResult("Using address %s for MicroCloud", listenAddr))
Expand All @@ -264,14 +264,19 @@ func (c *initConfig) askAddress(filterAddress string) error {
}

if subnet == nil {
return fmt.Errorf("Cloud not find valid subnet for address %q", listenAddr)
return nil, fmt.Errorf("Cloud not find valid subnet for address %q", listenAddr)
}

c.address = listenAddr
c.lookupIface = iface
c.lookupSubnet = subnet

return nil
_, cidrMicroCloudSubnet, err := net.ParseCIDR(c.lookupSubnet.String())
if err != nil {
return nil, fmt.Errorf("Failed to parse MicroCloud internal CIDR subnet: %w", err)
}

return &Network{Interface: *iface, Subnet: cidrMicroCloudSubnet, IP: net.IP(listenAddr)}, nil
}

func (c *initConfig) askDisks(sh *service.Handler) error {
Expand Down Expand Up @@ -1149,7 +1154,7 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
}
}

var ovnUnderlaySelectedIPs map[string]string
var ovnUnderlaySelectedNets map[string]*Network
ovnUnderlayData := [][]string{}
for peer, system := range c.systems {
// skip any systems that have already been clustered, but are available for other configuration.
Expand All @@ -1175,28 +1180,29 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {

if wantsDedicatedUnderlay {
header = []string{"LOCATION", "IFACE", "TYPE", "IP ADDRESS (CIDR)"}
ovnUnderlaySelectedIPs = map[string]string{}
err = c.askRetry("Retry selecting underlay network interfaces?", func() error {
table := tui.NewSelectableTable(header, ovnUnderlayData)
answers, err := table.Render(context.Background(), c.asker, "Select exactly one network interface from each cluster member:")
if err != nil {
return err
}

ovnUnderlaySelectedIPs = map[string]string{}
ovnUnderlaySelectedNets = make(map[string]*Network)
for _, answer := range answers {
target := answer["LOCATION"]
ipAddr := answer["IP ADDRESS (CIDR)"]
if ovnUnderlaySelectedIPs[target] != "" {
ifaceName := answer["IFACE"]

if ovnUnderlaySelectedNets[target] != nil {
return fmt.Errorf("Failed to configure OVN underlay traffic: Selected more than one interface for target %q", target)
}

ip, _, err := net.ParseCIDR(ipAddr)
ip, ipNet, err := net.ParseCIDR(ipAddr)
if err != nil {
return err
}

ovnUnderlaySelectedIPs[target] = ip.String()
ovnUnderlaySelectedNets[target] = &Network{Interface: net.Interface{Name: ifaceName}, IP: ip, Subnet: ipNet}
}

return nil
Expand All @@ -1207,11 +1213,11 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
}
}

if len(ovnUnderlaySelectedIPs) > 0 {
if len(ovnUnderlaySelectedNets) > 0 {
for peer := range askSystems {
underlayIP, ok := ovnUnderlaySelectedIPs[peer]
underlayNetwork, ok := ovnUnderlaySelectedNets[peer]
if ok {
fmt.Printf(" Using %q for OVN underlay traffic on %q\n", underlayIP, peer)
fmt.Printf(" Using %q for OVN underlay traffic on %q\n", underlayNetwork.IP.String(), peer)
}
}

Expand Down Expand Up @@ -1248,10 +1254,10 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
system.Networks = append(system.Networks, finalConfigs...)
}

if ovnUnderlaySelectedIPs != nil {
ovnUnderlayIpAddr, ok := ovnUnderlaySelectedIPs[peer]
if ovnUnderlaySelectedNets != nil {
ovnUnderlayNet, ok := ovnUnderlaySelectedNets[peer]
if ok {
system.OVNGeneveAddr = ovnUnderlayIpAddr
system.OVNGeneveNetwork = ovnUnderlayNet
}
}

Expand Down Expand Up @@ -1409,22 +1415,41 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error {
return err
}

internalCephNetworkInterface, err := lxd.FindInterfaceForSubnet(internalCephSubnet)
if err != nil {
return fmt.Errorf("Failed to find interface for subnet %q: %w", internalCephSubnet, err)
}

if internalCephSubnet != microCloudInternalNetworkAddrCIDR {
err = c.validateCephInterfacesForSubnet(lxd, availableCephNetworkInterfaces, internalCephSubnet)
if err != nil {
return err
}

bootstrapSystem := c.systems[sh.Name]
bootstrapSystem.MicroCephInternalNetworkSubnet = internalCephSubnet
c.systems[sh.Name] = bootstrapSystem
internalCephIP, internalCephNet, err := net.ParseCIDR(internalCephSubnet)
if err != nil {
return fmt.Errorf("Failed to parse the internal Ceph network: %w", err)
}

bootstrapSystem := c.systems[c.name]
bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: *internalCephNetworkInterface, Subnet: internalCephNet, IP: internalCephIP}
c.systems[c.name] = bootstrapSystem
} else {
bootstrapSystem := c.systems[c.name]
bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: bootstrapSystem.MicroCloudInternalNetwork.Interface, Subnet: bootstrapSystem.MicroCloudInternalNetwork.Subnet, IP: bootstrapSystem.MicroCloudInternalNetwork.IP}
c.systems[c.name] = bootstrapSystem
}

publicCephSubnet, err := c.asker.AskString("What subnet (either IPv4 or IPv6 CIDR notation) would you like your Ceph public traffic on?", internalCephSubnet, validate.IsNetwork)
if err != nil {
return err
}

publicCephNetworkInterface, err := lxd.FindInterfaceForSubnet(publicCephSubnet)
if err != nil {
return fmt.Errorf("Failed to find interface for subnet %q: %w", publicCephSubnet, err)
}

if publicCephSubnet != internalCephSubnet {
err = c.validateCephInterfacesForSubnet(lxd, availableCephNetworkInterfaces, publicCephSubnet)
if err != nil {
Expand All @@ -1433,17 +1458,31 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error {
}

if publicCephSubnet != microCloudInternalNetworkAddrCIDR {
publicCephIP, publicCephNet, err := net.ParseCIDR(publicCephSubnet)
if err != nil {
return fmt.Errorf("Failed to parse the public Ceph network: %w", err)
}

bootstrapSystem := c.systems[sh.Name]
bootstrapSystem.MicroCephPublicNetworkSubnet = publicCephSubnet
bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: *publicCephNetworkInterface, Subnet: publicCephNet, IP: publicCephIP}
c.systems[sh.Name] = bootstrapSystem

// This is to avoid the situation where the internal network for Ceph has been skipped, but the public network has been set.
// Ceph will automatically set the internal network to the public Ceph network if the internal network is not set, which is not what we want.
// Instead, we still want to keep the internal Ceph network to use the MicroCloud internal network as a default.
if internalCephSubnet == microCloudInternalNetworkAddrCIDR {
bootstrapSystem.MicroCephInternalNetworkSubnet = microCloudInternalNetworkAddrCIDR
microcloudInternalIP, microcloudNet, err := net.ParseCIDR(microCloudInternalNetworkAddrCIDR)
if err != nil {
return fmt.Errorf("Failed to parse the internal MicroCloud network: %w", err)
}

bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: bootstrapSystem.MicroCloudInternalNetwork.Interface, Subnet: microcloudNet, IP: microcloudInternalIP}
c.systems[sh.Name] = bootstrapSystem
}
} else {
bootstrapSystem := c.systems[sh.Name]
bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: bootstrapSystem.MicroCloudInternalNetwork.Interface, Subnet: bootstrapSystem.MicroCloudInternalNetwork.Subnet, IP: bootstrapSystem.MicroCloudInternalNetwork.IP}
c.systems[sh.Name] = bootstrapSystem
}

return nil
Expand Down
12 changes: 8 additions & 4 deletions cmd/microcloud/join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,16 +65,20 @@ func (c *cmdJoin) Run(cmd *cobra.Command, args []string) error {
cfg.sessionTimeout = time.Duration(c.flagSessionTimeout) * time.Second
}

err = cfg.askAddress(c.flagInitiatorAddress)
cfg.name, err = os.Hostname()
if err != nil {
return err
return fmt.Errorf("Failed to retrieve system hostname: %w", err)
}

cfg.name, err = os.Hostname()
microcloudInternalNet, err := cfg.askAddress(c.flagInitiatorAddress)
if err != nil {
return fmt.Errorf("Failed to retrieve system hostname: %w", err)
return err
}

joiningSystem := cfg.systems[cfg.name]
joiningSystem.MicroCloudInternalNetwork = microcloudInternalNet
cfg.systems[cfg.name] = joiningSystem

installedServices := []types.ServiceType{types.MicroCloud, types.LXD}
optionalServices := map[types.ServiceType]string{
types.MicroCeph: api.MicroCephDir,
Expand Down
42 changes: 24 additions & 18 deletions cmd/microcloud/main_init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,24 @@ type InitSystem struct {
AvailableDisks []lxdAPI.ResourcesStorageDisk
// MicroCephDisks contains the disks intended to be passed to MicroCeph.
MicroCephDisks []cephTypes.DisksPost
// MicroCephPublicNetworkSubnet is an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph public network.
MicroCephPublicNetworkSubnet string
// MicroCephClusterNetworkSubnet is an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph cluster network.
MicroCephInternalNetworkSubnet string
// MicroCephPublicNetwork contains an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph public network and
// the network interface name to use for the Ceph public network and its IP address within the subnet.
MicroCephPublicNetwork *Network
// MicroCephInternalNetwork contains an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph cluster network and
// the network interface name to use for the Ceph cluster network and its IP address within the subnet.
MicroCephInternalNetwork *Network
// MicroCloudInternalNetwork contains the network configuration for the MicroCloud internal network.
MicroCloudInternalNetwork *Network
// TargetNetworks contains the network configuration for the target system.
TargetNetworks []lxdAPI.NetworksPost
// TargetStoragePools contains the storage pool configuration for the target system.
TargetStoragePools []lxdAPI.StoragePoolsPost
// Networks is the cluster-wide network configuration.
Networks []lxdAPI.NetworksPost
// OVNGeneveAddr represents an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system.
// OVNGeneveNetwork contains an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system.
// If left empty, the system will choose to route the Geneve traffic through the management network.
OVNGeneveAddr string
// It also contains the network interface name to use for the OVN Geneve tunnel and the network subnet.
OVNGeneveNetwork *Network
// StoragePools is the cluster-wide storage pool configuration.
StoragePools []lxdAPI.StoragePoolsPost
// StorageVolumes is the cluster-wide storage volume configuration.
Expand Down Expand Up @@ -170,21 +175,22 @@ func (c *initConfig) RunInteractive(cmd *cobra.Command, args []string) error {
return err
}

err = c.askAddress("")
c.name, err = os.Hostname()
if err != nil {
return err
return fmt.Errorf("Failed to retrieve system hostname: %w", err)
}

c.name, err = os.Hostname()
microcloudInternalNet, err := c.askAddress("")
if err != nil {
return fmt.Errorf("Failed to retrieve system hostname: %w", err)
return err
}

c.systems[c.name] = InitSystem{
ServerInfo: multicast.ServerInfo{
Name: c.name,
Address: c.address,
},
MicroCloudInternalNetwork: microcloudInternalNet,
}

installedServices := []types.ServiceType{types.MicroCloud, types.LXD}
Expand Down Expand Up @@ -388,9 +394,9 @@ func (c *initConfig) addPeers(sh *service.Handler) (revert.Hook, error) {
CephConfig: info.MicroCephDisks,
}

if info.OVNGeneveAddr != "" {
if info.OVNGeneveNetwork != nil {
p := joinConfig[peer]
p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveAddr}
p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveNetwork.IP.String()}
joinConfig[peer] = p
}
}
Expand Down Expand Up @@ -694,12 +700,12 @@ func (c *initConfig) setupCluster(s *service.Handler) error {

if s.Type() == types.MicroCeph {
microCephBootstrapConf := make(map[string]string)
if bootstrapSystem.MicroCephInternalNetworkSubnet != "" {
microCephBootstrapConf["ClusterNet"] = bootstrapSystem.MicroCephInternalNetworkSubnet
if bootstrapSystem.MicroCephInternalNetwork != nil {
microCephBootstrapConf["ClusterNet"] = bootstrapSystem.MicroCephInternalNetwork.Subnet.String()
}

if bootstrapSystem.MicroCephPublicNetworkSubnet != "" {
microCephBootstrapConf["PublicNet"] = bootstrapSystem.MicroCephPublicNetworkSubnet
if bootstrapSystem.MicroCephPublicNetwork != nil {
microCephBootstrapConf["PublicNet"] = bootstrapSystem.MicroCephPublicNetwork.Subnet.String()
}

if len(microCephBootstrapConf) > 0 {
Expand All @@ -709,8 +715,8 @@ func (c *initConfig) setupCluster(s *service.Handler) error {

if s.Type() == types.MicroOVN {
microOvnBootstrapConf := make(map[string]string)
if bootstrapSystem.OVNGeneveAddr != "" {
microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveAddr
if bootstrapSystem.OVNGeneveNetwork != nil {
microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveNetwork.IP.String()
}

if len(microOvnBootstrapConf) > 0 {
Expand Down
Loading

0 comments on commit 64fdc39

Please sign in to comment.