diff --git a/docs/loadbalancer-annotations.md b/docs/loadbalancer-annotations.md index d14e721..6d59532 100644 --- a/docs/loadbalancer-annotations.md +++ b/docs/loadbalancer-annotations.md @@ -67,7 +67,7 @@ The default value is `5`. ### `service.beta.kubernetes.io/scw-loadbalancer-health-check-http-uri` This is the annotation to set the URI that is used by the `http` health check. -It is possible to set the uri per port, like `80:/;443,8443:/healthz`. +It is possible to set the uri per port, like `80:/;443,8443:mydomain.tld/healthz`. NB: Required when setting service.beta.kubernetes.io/scw-loadbalancer-health-check-type to `http` or `https`. ### `service.beta.kubernetes.io/scw-loadbalancer-health-check-http-method` @@ -128,8 +128,8 @@ This is the annotation that modifes what occurs when a backend server is marked The default value is `on_marked_down_action_none` and the possible values are `on_marked_down_action_none` and `shutdown_sessions`. ### `service.beta.kubernetes.io/scw-loadbalancer-force-internal-ip` -This is the annotation that force the usage of InternalIP inside the loadbalancer. -Normally, the cloud controller manager use ExternalIP to be nodes region-free (or public InternalIP in case of Baremetal). +**This field is DEPRECATED**. This annotation is deprecated and will be removed in a future release. +It used to make the CCM use internal IPs instead of public ones for Public only clusters. ### `service.beta.kubernetes.io/scw-loadbalancer-use-hostname` This is the annotation that force the use of the LB hostname instead of the public IP. @@ -153,7 +153,7 @@ Expected format: `"Key1=Val1,Key2=Val2"` ### `service.beta.kubernetes.io/scw-loadbalancer-redispatch-attempt-count` This is the annotation to activate redispatch on another backend server in case of failure -The default value is 0, which disable the redispatch. +The default value is 0, which disable the redispatch. Only a value of 0 or 1 are allowed. ### `service.beta.kubernetes.io/scw-loadbalancer-max-retries` This is the annotation to configure the number of retry on connection failure diff --git a/scaleway/instances.go b/scaleway/instances.go index 17bed00..6ea04bf 100644 --- a/scaleway/instances.go +++ b/scaleway/instances.go @@ -58,7 +58,7 @@ func (i *instances) NodeAddresses(ctx context.Context, name types.NodeName) ([]v if err != nil { return nil, err } - return i.instanceAddresses(server), nil + return i.instanceAddresses(server) } // NodeAddressesByProviderID returns the addresses of the specified instance. @@ -68,7 +68,7 @@ func (i *instances) NodeAddressesByProviderID(ctx context.Context, providerID st if err != nil { return nil, err } - return i.instanceAddresses(instanceServer), nil + return i.instanceAddresses(instanceServer) } // InstanceID returns the cloud provider ID of the node with the specified NodeName. @@ -161,7 +161,7 @@ func (i *instances) GetZoneByNodeName(ctx context.Context, nodeName types.NodeNa // =========================== // instanceAddresses extracts NodeAdress from the server -func (i *instances) instanceAddresses(server *scwinstance.Server) []v1.NodeAddress { +func (i *instances) instanceAddresses(server *scwinstance.Server) ([]v1.NodeAddress, error) { addresses := []v1.NodeAddress{ {Type: v1.NodeHostName, Address: server.Hostname}, } @@ -194,13 +194,11 @@ func (i *instances) instanceAddresses(server *scwinstance.Server) []v1.NodeAddre Region: region, }) if err != nil { - klog.Errorf("unable to query ipam for node %s: %v", server.Name, err) - return addresses + return addresses, fmt.Errorf("unable to query ipam for node %s: %v", server.Name, err) } if len(ips.IPs) == 0 { - klog.Errorf("no private network ip for node %s", server.Name) - return addresses + return addresses, fmt.Errorf("no private network ip for node %s", server.Name) } for _, nicIP := range ips.IPs { @@ -210,7 +208,7 @@ func (i *instances) instanceAddresses(server *scwinstance.Server) []v1.NodeAddre ) } - return addresses + return addresses, nil } // fallback to legacy private ip @@ -222,7 +220,7 @@ func (i *instances) instanceAddresses(server *scwinstance.Server) []v1.NodeAddre ) } - return addresses + return addresses, nil } func instanceZone(instanceServer *scwinstance.Server) (cloudprovider.Zone, error) { @@ -369,10 +367,15 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud return nil, err } + addresses, err := i.instanceAddresses(instance) + if err != nil { + return nil, err + } + return &cloudprovider.InstanceMetadata{ ProviderID: BuildProviderID(InstanceTypeInstance, instance.Zone.String(), instance.ID), InstanceType: instance.CommercialType, - NodeAddresses: i.instanceAddresses(instance), + NodeAddresses: addresses, Region: region.String(), Zone: instance.Zone.String(), }, nil diff --git a/scaleway/loadbalancers.go b/scaleway/loadbalancers.go index 89ca499..003e9b7 100644 --- a/scaleway/loadbalancers.go +++ b/scaleway/loadbalancers.go @@ -19,13 +19,15 @@ package scaleway import ( "context" "fmt" - "math" "net" + "net/url" "os" + "reflect" "strconv" "strings" "time" + "golang.org/x/exp/slices" "google.golang.org/protobuf/types/known/durationpb" v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" @@ -78,7 +80,7 @@ const ( serviceAnnotationLoadBalancerHealthCheckMaxRetries = "service.beta.kubernetes.io/scw-loadbalancer-health-check-max-retries" // serviceAnnotationLoadBalancerHealthCheckHTTPURI is the URI that is used by the "http" health check - // It is possible to set the uri per port, like "80:/;443,8443:/healthz" + // It is possible to set the uri per port, like "80:/;443,8443:mydomain.tld/healthz" // NB: Required when setting service.beta.kubernetes.io/scw-loadbalancer-health-check-type to "http" or "https" serviceAnnotationLoadBalancerHealthCheckHTTPURI = "service.beta.kubernetes.io/scw-loadbalancer-health-check-http-uri" @@ -630,6 +632,7 @@ func (l *loadbalancers) updateLoadBalancer(ctx context.Context, loadbalancer *sc } // this PN should not be attached to this loadbalancer + klog.V(3).Infof("detach extra private network %s from load balancer %s", pNIC.PrivateNetworkID, loadbalancer.ID) err = l.api.DetachPrivateNetwork(&scwlb.ZonedAPIDetachPrivateNetworkRequest{ Zone: loadbalancer.Zone, LBID: loadbalancer.ID, @@ -641,6 +644,7 @@ func (l *loadbalancers) updateLoadBalancer(ctx context.Context, loadbalancer *sc } if pnNIC == nil { + klog.V(3).Infof("attach private network %s to load balancer %s", l.pnID, loadbalancer.ID) _, err = l.api.AttachPrivateNetwork(&scwlb.ZonedAPIAttachPrivateNetworkRequest{ Zone: loadbalancer.Zone, LBID: loadbalancer.ID, @@ -648,277 +652,170 @@ func (l *loadbalancers) updateLoadBalancer(ctx context.Context, loadbalancer *sc DHCPConfig: &scwlb.PrivateNetworkDHCPConfig{}, }) if err != nil { - return fmt.Errorf("unable to attach private network on %s: %v", loadbalancer.ID, err) + return fmt.Errorf("unable to attach private network %s on %s: %v", l.pnID, loadbalancer.ID, err) } } } + var targetIPs []string + if getForceInternalIP(service) || l.pnID != "" { + targetIPs = extractNodesInternalIps(nodes) + klog.V(3).Infof("using internal nodes ips: %s on loadbalancer %s", strings.Join(targetIPs, ","), loadbalancer.ID) + } else { + targetIPs = extractNodesExternalIps(nodes) + klog.V(3).Infof("using external nodes ips: %s on loadbalancer %s", strings.Join(targetIPs, ","), loadbalancer.ID) + } + // List all frontends associated with the LB respFrontends, err := l.api.ListFrontends(&scwlb.ZonedAPIListFrontendsRequest{ Zone: loadbalancer.Zone, LBID: loadbalancer.ID, }, scw.WithAllPages()) - if err != nil { return fmt.Errorf("error listing frontends for load balancer %s: %v", loadbalancer.ID, err) } - frontends := respFrontends.Frontends - - portFrontends := make(map[int32]*scwlb.Frontend) - for _, frontend := range frontends { - keep := false - for _, port := range service.Spec.Ports { - // if the frontend is still valid keep it - if port.Port == frontend.InboundPort && port.NodePort == frontend.Backend.ForwardPort { - keep = true - break - } - } - - if !keep { - // if the frontend is not valid anymore, delete it - klog.Infof("deleting frontend: %s", frontend.ID) - - err := l.api.DeleteFrontend(&scwlb.ZonedAPIDeleteFrontendRequest{ - Zone: loadbalancer.Zone, - FrontendID: frontend.ID, - }) - - if err != nil { - return fmt.Errorf("error deleting frontend %s: %v", frontend.ID, err) - } - } else { - portFrontends[frontend.InboundPort] = frontend - } - } - // List all backends associated with the LB respBackends, err := l.api.ListBackends(&scwlb.ZonedAPIListBackendsRequest{ Zone: loadbalancer.Zone, LBID: loadbalancer.ID, }, scw.WithAllPages()) - if err != nil { return fmt.Errorf("error listing backend for load balancer %s: %v", loadbalancer.ID, err) } - backends := respBackends.Backends + svcFrontends, svcBackends, err := serviceToLB(service, loadbalancer, targetIPs) + if err != nil { + return fmt.Errorf("failed to convert service to frontend and backends on loadbalancer %s: %v", loadbalancer.ID, err) + } - portBackends := make(map[int32]*scwlb.Backend) - for _, backend := range backends { - keep := false - for _, port := range service.Spec.Ports { - // if the backend is still valid, keep it - if port.NodePort == backend.ForwardPort { - keep = true - break - } - } + frontendsOps := compareFrontends(respFrontends.Frontends, svcFrontends) + backendsOps := compareBackends(respBackends.Backends, svcBackends) - if !keep { - // if the backend is not valid, delete it - err := l.api.DeleteBackend(&scwlb.ZonedAPIDeleteBackendRequest{ - Zone: loadbalancer.Zone, - BackendID: backend.ID, - }) + // Remove extra frontends + for _, f := range frontendsOps.remove { + klog.V(3).Infof("deleting frontend: %s port: %d loadbalancer: %s", f.ID, f.InboundPort, loadbalancer.ID) + if err := l.api.DeleteFrontend(&scwlb.ZonedAPIDeleteFrontendRequest{ + Zone: loadbalancer.Zone, + FrontendID: f.ID, + }); err != nil { + return fmt.Errorf("failed deleting frontend: %s port: %d loadbalancer: %s err: %v", f.ID, f.InboundPort, loadbalancer.ID, err) + } + } - if err != nil { - return fmt.Errorf("error deleting backend %s: %v", backend.ID, err) - } - } else { - portBackends[backend.ForwardPort] = backend + // Remove extra backends + for _, b := range backendsOps.remove { + klog.V(3).Infof("deleting backend: %s port: %d loadbalancer: %s", b.ID, b.ForwardPort, loadbalancer.ID) + if err := l.api.DeleteBackend(&scwlb.ZonedAPIDeleteBackendRequest{ + Zone: loadbalancer.Zone, + BackendID: b.ID, + }); err != nil { + return fmt.Errorf("failed deleting backend: %s port: %d loadbalancer: %s err: %v", b.ID, b.ForwardPort, loadbalancer.ID, err) } } - // loop through all the service ports for _, port := range service.Spec.Ports { - // if the corresponding backend exists for the node port, update it - if backend, ok := portBackends[port.NodePort]; ok { - updateBackendRequest, err := l.makeUpdateBackendRequest(backend, service, nodes) - if err != nil { - klog.Errorf("error making UpdateBackendRequest: %v", err) - return err - } - - updateBackendRequest.Zone = loadbalancer.Zone - updateBackendRequest.ForwardPort = port.NodePort - _, err = l.api.UpdateBackend(updateBackendRequest) - if err != nil { - klog.Errorf("error updating backend %s: %v", backend.ID, err) - return fmt.Errorf("error updating backend %s: %v", backend.ID, err) - } - - updateHealthCheckRequest, err := l.makeUpdateHealthCheckRequest(backend, port.NodePort, service, nodes) + var backend *scwlb.Backend + // Update backend + if b, ok := backendsOps.update[port.NodePort]; ok { + klog.V(3).Infof("update backend: %s port: %d loadbalancer: %s", b.ID, b.ForwardPort, loadbalancer.ID) + updatedBackend, err := l.updateBackend(service, loadbalancer, b) if err != nil { - klog.Errorf("error making UpdateHealthCheckRequest: %v", err) - return err + return fmt.Errorf("failed updating backend %s port: %d loadbalancer: %s err: %v", b.ID, b.ForwardPort, loadbalancer.ID, err) } - - updateHealthCheckRequest.Zone = loadbalancer.Zone - _, err = l.api.UpdateHealthCheck(updateHealthCheckRequest) + backend = updatedBackend + } + // Create backend + if b, ok := backendsOps.create[port.NodePort]; ok { + klog.V(3).Infof("create backend port: %d loadbalancer: %s", b.ForwardPort, loadbalancer.ID) + createdBackend, err := l.createBackend(service, loadbalancer, b) if err != nil { - klog.Errorf("error updating healthcheck for backend %s: %v", backend.ID, err) - return fmt.Errorf("error updating healthcheck for backend %s: %v", backend.ID, err) + return fmt.Errorf("failed creating backend port: %d loadbalancer: %s err: %v", b.ForwardPort, loadbalancer.ID, err) } + backend = createdBackend + } - var serverIPs []string - if getForceInternalIP(service) || l.pnID != "" { - serverIPs = extractNodesInternalIps(nodes) - } else { - serverIPs = extractNodesExternalIps(nodes) + if backend == nil { + b, ok := backendsOps.keep[port.NodePort] + if !ok { + return fmt.Errorf("undefined backend port: %d loadbalancer: %s", port.NodePort, loadbalancer.ID) } + backend = b + } - setBackendServersRequest := &scwlb.ZonedAPISetBackendServersRequest{ + // Update backend servers + if !stringArrayEqual(backend.Pool, targetIPs) { + klog.V(3).Infof("update server list for backend: %s port: %d loadbalancer: %s", backend.ID, port.NodePort, loadbalancer.ID) + if _, err := l.api.SetBackendServers(&scwlb.ZonedAPISetBackendServersRequest{ Zone: loadbalancer.Zone, BackendID: backend.ID, - ServerIP: serverIPs, - } - - respBackend, err := l.api.SetBackendServers(setBackendServersRequest) - if err != nil { - klog.Errorf("error setting backend servers for backend %s: %v", backend.ID, err) - return fmt.Errorf("error setting backend servers for backend %s: %v", backend.ID, err) - } - - portBackends[backend.ForwardPort] = respBackend - } else { // if a backend does not exists for the node port, create it - request, err := l.makeCreateBackendRequest(loadbalancer, port.NodePort, service, nodes) - if err != nil { - klog.Errorf("error making CreateBackendRequest: %v", err) - return err + ServerIP: targetIPs, + }); err != nil { + return fmt.Errorf("failed updating server list for backend: %s port: %d loadbalancer: %s err: %v", backend.ID, port.NodePort, loadbalancer.ID, err) } + } - respBackend, err := l.api.CreateBackend(request) + var frontend *scwlb.Frontend + // Update frontend + if f, ok := frontendsOps.update[port.Port]; ok { + klog.V(3).Infof("update frontend: %s port: %d loadbalancer: %s", f.ID, port.Port, loadbalancer.ID) + ff, err := l.updateFrontend(service, loadbalancer, f, backend) if err != nil { - klog.Errorf("error creating backend on load balancer %s: %v", loadbalancer.ID, err) - return fmt.Errorf("error creating backend on load balancer %s: %v", loadbalancer.ID, err) + return fmt.Errorf("failed updating frontend: %s port: %d loadbalancer: %s err: %v", f.ID, port.Port, loadbalancer.ID, err) } - - portBackends[port.NodePort] = respBackend - } - } - - for _, port := range service.Spec.Ports { - var frontendID string - certificateIDs, err := getCertificateIDs(service, port.Port) - if err != nil { - return fmt.Errorf("error getting certificate IDs for loadbalancer %s: %v", loadbalancer.ID, err) + frontend = ff } - - timeoutClient, err := getTimeoutClient(service) - if err != nil { - return fmt.Errorf("error getting %s annotation for loadbalancer %s: %v", - serviceAnnotationLoadBalancerTimeoutClient, loadbalancer.ID, err) - } - - // if the frontend exists for the port, update it - if frontend, ok := portFrontends[port.Port]; ok { - _, err := l.api.UpdateFrontend(&scwlb.ZonedAPIUpdateFrontendRequest{ - Zone: loadbalancer.Zone, - FrontendID: frontend.ID, - Name: frontend.Name, - InboundPort: frontend.InboundPort, - BackendID: portBackends[port.NodePort].ID, - TimeoutClient: &timeoutClient, - CertificateIDs: scw.StringsPtr(certificateIDs), - }) - + // Create frontend + if f, ok := frontendsOps.create[port.Port]; ok { + klog.V(3).Infof("create frontend port: %d loadbalancer: %s", port.Port, loadbalancer.ID) + ff, err := l.createFrontend(service, loadbalancer, f, backend) if err != nil { - klog.Errorf("error updating frontend %s: %v", frontend.ID, err) - return fmt.Errorf("error updating frontend %s: %v", frontend.ID, err) + return fmt.Errorf("failed creating frontend port: %d loadbalancer: %s err: %v", port.Port, loadbalancer.ID, err) } + frontend = ff + } - frontendID = frontend.ID - } else { // if the frontend for this port does not exist, create it - resp, err := l.api.CreateFrontend(&scwlb.ZonedAPICreateFrontendRequest{ - Zone: loadbalancer.Zone, - LBID: loadbalancer.ID, - Name: fmt.Sprintf("%s_tcp_%d", string(service.UID), port.Port), - InboundPort: port.Port, - BackendID: portBackends[port.NodePort].ID, - TimeoutClient: &timeoutClient, - CertificateIDs: scw.StringsPtr(certificateIDs), - }) - - if err != nil { - klog.Errorf("error creating frontend on load balancer %s: %v", loadbalancer.ID, err) - return fmt.Errorf("error creating frontend on load balancer %s: %v", loadbalancer.ID, err) + if frontend == nil { + f, ok := frontendsOps.keep[port.Port] + if !ok { + return fmt.Errorf("undefined frontend port: %d loadbalancer: %s", port.Port, loadbalancer.ID) } - - frontendID = resp.ID + frontend = f } - aclName := frontendID + "-lb-source-range" - - acls, err := l.api.ListACLs(&scwlb.ZonedAPIListACLsRequest{ + // List ACLs for the frontend + aclName := makeACLPrefix(frontend) + aclsResp, err := l.api.ListACLs(&scwlb.ZonedAPIListACLsRequest{ Zone: loadbalancer.Zone, - FrontendID: frontendID, + FrontendID: frontend.ID, Name: &aclName, }, scw.WithAllPages()) if err != nil { - return err + return fmt.Errorf("failed to list ACLs for frontend: %s port: %d loadbalancer: %s err: %v", frontend.ID, frontend.InboundPort, loadbalancer.ID, err) } - if len(service.Spec.LoadBalancerSourceRanges) == 0 || len(acls.ACLs) != 1 { - for _, acl := range acls.ACLs { - err = l.api.DeleteACL(&scwlb.ZonedAPIDeleteACLRequest{ + svcAcls := makeACLSpecs(service, nodes, frontend) + if !aclsEquals(aclsResp.ACLs, svcAcls) { + // Replace ACLs + klog.Infof("remove all ACLs from frontend: %s port: %d loadbalancer: %s", frontend.ID, frontend.InboundPort, loadbalancer.ID) + for _, acl := range aclsResp.ACLs { + if err := l.api.DeleteACL(&scwlb.ZonedAPIDeleteACLRequest{ Zone: loadbalancer.Zone, ACLID: acl.ID, - }) - if err != nil { - return err + }); err != nil { + return fmt.Errorf("failed removing ACL %s from frontend: %s port: %d loadbalancer: %s err: %v", acl.Name, frontend.ID, frontend.InboundPort, loadbalancer.ID, err) } } - } - if len(service.Spec.LoadBalancerSourceRanges) != 0 { - aclIPs := extractNodesInternalIps(nodes) - aclIPs = append(aclIPs, extractNodesExternalIps(nodes)...) - aclIPs = append(aclIPs, service.Spec.LoadBalancerSourceRanges...) - aclIPsPtr := []*string{} - newAcls := []*scwlb.ACLSpec{} - - // Loop through all addresses and make sure to split ACLs every MaxEntriesPerACL. - for i := range aclIPs { - if i != 0 && i%MaxEntriesPerACL == 0 { - aclIndex := int32((i / MaxEntriesPerACL) - 1) - newAcls = append(newAcls, &scwlb.ACLSpec{ - Name: fmt.Sprintf("%v-%d", aclName, aclIndex), - Action: &scwlb.ACLAction{ - Type: scwlb.ACLActionTypeAllow, - }, - Index: aclIndex, - Match: &scwlb.ACLMatch{ - IPSubnet: aclIPsPtr, - }, - }) - - aclIPsPtr = []*string{} + klog.Infof("create all ACLs for frontend: %s port: %d loadbalancer: %s", frontend.ID, frontend.InboundPort, loadbalancer.ID) + for _, acl := range svcAcls { + if _, err := l.api.SetACLs(&scwlb.ZonedAPISetACLsRequest{ + Zone: loadbalancer.Zone, + FrontendID: frontend.ID, + ACLs: svcAcls, + }); err != nil { + return fmt.Errorf("failed creating ACL %s for frontend: %s port: %d loadbalancer: %s err: %v", acl.Name, frontend.ID, frontend.InboundPort, loadbalancer.ID, err) } - aclIPsPtr = append(aclIPsPtr, &aclIPs[i]) - } - - // Add last ACL with remaining addresses. - newAcls = append(newAcls, &scwlb.ACLSpec{ - Name: aclName + "-end", - Action: &scwlb.ACLAction{ - Type: scwlb.ACLActionTypeDeny, - }, - Index: math.MaxInt32, - Match: &scwlb.ACLMatch{ - IPSubnet: aclIPsPtr, - Invert: true, - }, - }) - - _, err := l.api.SetACLs(&scwlb.ZonedAPISetACLsRequest{ - Zone: loadbalancer.Zone, - FrontendID: frontendID, - ACLs: newAcls, - }) - if err != nil { - return err } } } @@ -939,1127 +836,1443 @@ func (l *loadbalancers) updateLoadBalancer(ctx context.Context, loadbalancer *sc return nil } -func (l *loadbalancers) makeUpdateBackendRequest(backend *scwlb.Backend, service *v1.Service, nodes []*v1.Node) (*scwlb.ZonedAPIUpdateBackendRequest, error) { - protocol, err := getForwardProtocol(service, backend.ForwardPort) - if err != nil { - return nil, err - } - - request := &scwlb.ZonedAPIUpdateBackendRequest{ - BackendID: backend.ID, - Name: backend.Name, - ForwardProtocol: protocol, +// createPrivateServiceStatus creates a LoadBalancer status for services with private load balancers +func (l *loadbalancers) createPrivateServiceStatus(service *v1.Service, lb *scwlb.LB) (*v1.LoadBalancerStatus, error) { + if l.pnID == "" { + return nil, fmt.Errorf("cannot make status for service %s/%s: private load balancer requires a private network", service.Namespace, service.Name) } - forwardPortAlgorithm, err := getForwardPortAlgorithm(service) + region, err := lb.Zone.Region() if err != nil { - return nil, err + return nil, fmt.Errorf("error making status for service %s/%s: %v", service.Namespace, service.Name, err) } - request.ForwardPortAlgorithm = forwardPortAlgorithm - stickySessions, err := getStickySessions(service) - if err != nil { - return nil, err - } + status := &v1.LoadBalancerStatus{} - request.StickySessions = stickySessions + if getUseHostname(service) { + pn, err := l.vpc.GetPrivateNetwork(&scwvpc.GetPrivateNetworkRequest{ + Region: region, + PrivateNetworkID: l.pnID, + }) + if err != nil { + return nil, fmt.Errorf("unable to query private network for lb %s: %v", lb.Name, err) + } - if stickySessions == scwlb.StickySessionsTypeCookie { - stickySessionsCookieName, err := getStickySessionsCookieName(service) + status.Ingress = []v1.LoadBalancerIngress{ + { + Hostname: fmt.Sprintf("%s.%s", lb.ID, pn.Name), + }, + } + } else { + ipamRes, err := l.ipam.ListIPs(&scwipam.ListIPsRequest{ + ProjectID: &lb.ProjectID, + ResourceType: scwipam.ResourceTypeLBServer, + ResourceID: &lb.ID, + IsIPv6: scw.BoolPtr(false), + Region: region, + }) if err != nil { - return nil, err + return nil, fmt.Errorf("unable to query ipam for lb %s: %v", lb.Name, err) } - if stickySessionsCookieName == "" { - klog.Errorf("missing annotation %s", serviceAnnotationLoadBalancerStickySessionsCookieName) - return nil, NewAnnorationError(serviceAnnotationLoadBalancerStickySessionsCookieName, stickySessionsCookieName) + + if len(ipamRes.IPs) == 0 { + return nil, fmt.Errorf("no private network ip for lb %s", lb.Name) } - request.StickySessionsCookieName = stickySessionsCookieName - } - proxyProtocol, err := getProxyProtocol(service, backend.ForwardPort) - if err != nil { - return nil, err + status.Ingress = make([]v1.LoadBalancerIngress, len(ipamRes.IPs)) + for idx, ip := range ipamRes.IPs { + status.Ingress[idx].IP = ip.Address.IP.String() + } } - request.ProxyProtocol = proxyProtocol - timeoutServer, err := getTimeoutServer(service) - if err != nil { - return nil, err - } + return status, nil +} - request.TimeoutServer = &timeoutServer +// createPublicServiceStatus creates a LoadBalancer status for services with public load balancers +func (l *loadbalancers) createPublicServiceStatus(service *v1.Service, lb *scwlb.LB) (*v1.LoadBalancerStatus, error) { + status := &v1.LoadBalancerStatus{} + status.Ingress = make([]v1.LoadBalancerIngress, 0) + for _, ip := range lb.IP { + // Skip ipv6 entries + if i := net.ParseIP(ip.IPAddress); i.To4() == nil { + continue + } - timeoutConnect, err := getTimeoutConnect(service) - if err != nil { - return nil, err + if getUseHostname(service) { + status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{Hostname: ip.Reverse}) + } else { + status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{IP: ip.IPAddress}) + } } - request.TimeoutConnect = &timeoutConnect - - timeoutTunnel, err := getTimeoutTunnel(service) - if err != nil { - return nil, err + if len(status.Ingress) == 0 { + return nil, fmt.Errorf("no ipv4 found for lb %s", lb.Name) } - request.TimeoutTunnel = &timeoutTunnel + return status, nil +} - onMarkedDownAction, err := getOnMarkedDownAction(service) +// createServiceStatus creates a LoadBalancer status for the service +func (l *loadbalancers) createServiceStatus(service *v1.Service, lb *scwlb.LB) (*v1.LoadBalancerStatus, error) { + lbPrivate, err := svcPrivate(service) if err != nil { - return nil, err + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerPrivate) + return nil, fmt.Errorf("invalid value for annotation %s: expected boolean", serviceAnnotationLoadBalancerPrivate) } - request.OnMarkedDownAction = onMarkedDownAction - - redispatchAttemptCount, err := getRedisatchAttemptCount(service) - if err != nil { - return nil, err + if lbPrivate { + return l.createPrivateServiceStatus(service, lb) } - request.RedispatchAttemptCount = redispatchAttemptCount + return l.createPublicServiceStatus(service, lb) +} + +func getLoadBalancerID(service *v1.Service) (scw.Zone, string, error) { + annoLoadBalancerID, ok := service.Annotations[serviceAnnotationLoadBalancerID] + if !ok { + return "", "", errLoadBalancerInvalidAnnotation + } - maxRetries, err := getMaxRetries(service) - if err != nil { - return nil, err + splitLoadBalancerID := strings.Split(strings.ToLower(annoLoadBalancerID), "/") + if len(splitLoadBalancerID) != 2 { + return "", "", errLoadBalancerInvalidLoadBalancerID } - request.MaxRetries = maxRetries + if validation.IsRegion(splitLoadBalancerID[0]) { + zone := splitLoadBalancerID[0] + "-1" + return scw.Zone(zone), splitLoadBalancerID[1], nil + } - return request, nil + return scw.Zone(splitLoadBalancerID[0]), splitLoadBalancerID[1], nil } -func (l *loadbalancers) makeUpdateHealthCheckRequest(backend *scwlb.Backend, nodePort int32, service *v1.Service, nodes []*v1.Node) (*scwlb.ZonedAPIUpdateHealthCheckRequest, error) { - request := &scwlb.ZonedAPIUpdateHealthCheckRequest{ - BackendID: backend.ID, - Port: nodePort, +func getForwardPortAlgorithm(service *v1.Service) (scwlb.ForwardPortAlgorithm, error) { + forwardPortAlgorithm, ok := service.Annotations[serviceAnnotationLoadBalancerForwardPortAlgorithm] + if !ok { + return scwlb.ForwardPortAlgorithmRoundrobin, nil } - healthCheckDelay, err := getHealthCheckDelay(service) - if err != nil { - return nil, err + forwardPortAlgorithmValue := scwlb.ForwardPortAlgorithm(forwardPortAlgorithm) + + if forwardPortAlgorithmValue != scwlb.ForwardPortAlgorithmRoundrobin && forwardPortAlgorithmValue != scwlb.ForwardPortAlgorithmLeastconn && forwardPortAlgorithmValue != scwlb.ForwardPortAlgorithmFirst { + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerForwardPortAlgorithm) + return "", errLoadBalancerInvalidAnnotation } - request.CheckDelay = &healthCheckDelay + return forwardPortAlgorithmValue, nil +} - healthCheckTimeout, err := getHealthCheckTimeout(service) - if err != nil { - return nil, err +func getStickySessions(service *v1.Service) (scwlb.StickySessionsType, error) { + stickySessions, ok := service.Annotations[serviceAnnotationLoadBalancerStickySessions] + if !ok { + return scwlb.StickySessionsTypeNone, nil } - request.CheckTimeout = &healthCheckTimeout + stickySessionsValue := scwlb.StickySessionsType(stickySessions) - healthCheckMaxRetries, err := getHealthCheckMaxRetries(service) - if err != nil { - return nil, err + if stickySessionsValue != scwlb.StickySessionsTypeNone && stickySessionsValue != scwlb.StickySessionsTypeCookie && stickySessionsValue != scwlb.StickySessionsTypeTable { + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerStickySessions) + return "", errLoadBalancerInvalidAnnotation } - request.CheckMaxRetries = healthCheckMaxRetries + return stickySessionsValue, nil +} - transientCheckDelay, err := getHealthCheckTransientCheckDelay(service) - if err != nil { - return nil, err +func getStickySessionsCookieName(service *v1.Service) (string, error) { + stickySessionsCookieName, ok := service.Annotations[serviceAnnotationLoadBalancerStickySessionsCookieName] + if !ok { + return "", nil } - request.TransientCheckDelay = transientCheckDelay + return stickySessionsCookieName, nil +} - healthCheckType, err := getHealthCheckType(service, nodePort) - if err != nil { - return nil, err +func getSendProxyV2(service *v1.Service, nodePort int32) (scwlb.ProxyProtocol, error) { + sendProxyV2, ok := service.Annotations[serviceAnnotationLoadBalancerSendProxyV2] + if !ok { + return scwlb.ProxyProtocolProxyProtocolNone, nil } - switch healthCheckType { - case "mysql": - hc, err := getMysqlHealthCheck(service, nodePort) - if err != nil { - return nil, err + sendProxyV2Value, err := strconv.ParseBool(sendProxyV2) + if err != nil { + var svcPort int32 = -1 + for _, p := range service.Spec.Ports { + if p.NodePort == nodePort { + svcPort = p.Port + } } - request.MysqlConfig = hc - case "ldap": - hc, err := getLdapHealthCheck(service, nodePort) - if err != nil { - return nil, err + if svcPort == -1 { + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerSendProxyV2) + return "", errLoadBalancerInvalidAnnotation } - request.LdapConfig = hc - case "redis": - hc, err := getRedisHealthCheck(service, nodePort) - if err != nil { - return nil, err - } - request.RedisConfig = hc - case "pgsql": - hc, err := getPgsqlHealthCheck(service, nodePort) - if err != nil { - return nil, err - } - request.PgsqlConfig = hc - case "tcp": - hc, err := getTCPHealthCheck(service, nodePort) - if err != nil { - return nil, err + + ports := strings.Split(strings.ReplaceAll(sendProxyV2, " ", ""), ",") + for _, port := range ports { + intPort, err := strconv.ParseInt(port, 0, 64) + if err != nil { + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerSendProxyV2) + return "", errLoadBalancerInvalidAnnotation + } + if int64(svcPort) == intPort { + return scwlb.ProxyProtocolProxyProtocolV2, nil + } } - request.TCPConfig = hc - case "http": - hc, err := getHTTPHealthCheck(service, nodePort) + return scwlb.ProxyProtocolProxyProtocolNone, nil + } + + if sendProxyV2Value { + return scwlb.ProxyProtocolProxyProtocolV2, nil + } + + return scwlb.ProxyProtocolProxyProtocolNone, nil +} + +func isPortInRange(r string, p int32) (bool, error) { + boolValue, err := strconv.ParseBool(r) + if err == nil && r != "1" && r != "0" { + return boolValue, nil + } + if r == "*" { + return true, nil + } + if r == "" { + return false, nil + } + ports := strings.Split(strings.ReplaceAll(r, " ", ""), ",") + for _, port := range ports { + intPort, err := strconv.ParseInt(port, 0, 64) if err != nil { - return nil, err + return false, err } - request.HTTPConfig = hc - case "https": - hc, err := getHTTPSHealthCheck(service, nodePort) - if err != nil { - return nil, err + if int64(p) == intPort { + return true, nil } - request.HTTPSConfig = hc - default: - klog.Errorf("wrong value for healthCheckType") - return nil, NewAnnorationError(serviceAnnotationLoadBalancerHealthCheckType, healthCheckType) } + return false, nil +} - return request, nil +func getLoadBalancerType(service *v1.Service) string { + return strings.ToLower(service.Annotations[serviceAnnotationLoadBalancerType]) } -func (l *loadbalancers) makeCreateBackendRequest(loadbalancer *scwlb.LB, nodePort int32, service *v1.Service, nodes []*v1.Node) (*scwlb.ZonedAPICreateBackendRequest, error) { - protocol, err := getForwardProtocol(service, nodePort) +func getLoadBalancerZone(service *v1.Service) scw.Zone { + return scw.Zone(strings.ToLower(service.Annotations[serviceAnnotationLoadBalancerZone])) +} + +func getProxyProtocol(service *v1.Service, nodePort int32) (scwlb.ProxyProtocol, error) { + proxyProtocolV1 := service.Annotations[serviceAnnotationLoadBalancerProxyProtocolV1] + proxyProtocolV2 := service.Annotations[serviceAnnotationLoadBalancerProxyProtocolV2] + + var svcPort int32 = -1 + for _, p := range service.Spec.Ports { + if p.NodePort == nodePort { + svcPort = p.Port + } + } + if svcPort == -1 { + klog.Errorf("no valid port found") + return "", errLoadBalancerInvalidAnnotation + } + + isV1, err := isPortInRange(proxyProtocolV1, svcPort) if err != nil { - return nil, err + klog.Errorf("unable to check if port %d is in range %s", svcPort, proxyProtocolV1) + return "", err } - var serverIPs []string - if getForceInternalIP(service) || l.pnID != "" { - serverIPs = extractNodesInternalIps(nodes) - } else { - serverIPs = extractNodesExternalIps(nodes) + isV2, err := isPortInRange(proxyProtocolV2, svcPort) + if err != nil { + klog.Errorf("unable to check if port %d is in range %s", svcPort, proxyProtocolV2) + return "", err } - request := &scwlb.ZonedAPICreateBackendRequest{ - Zone: loadbalancer.Zone, - LBID: loadbalancer.ID, - Name: fmt.Sprintf("%s_tcp_%d", string(service.UID), nodePort), - ServerIP: serverIPs, - ForwardProtocol: protocol, - ForwardPort: nodePort, + + if isV1 && isV2 { + klog.Errorf("port %d is in both v1 and v2 proxy protocols", svcPort) + return "", fmt.Errorf("port %d is in both v1 and v2 proxy protocols", svcPort) } - forwardPortAlgorithm, err := getForwardPortAlgorithm(service) - if err != nil { - return nil, err + if isV1 { + return scwlb.ProxyProtocolProxyProtocolV1, nil + } + if isV2 { + return scwlb.ProxyProtocolProxyProtocolV2, nil } - request.ForwardPortAlgorithm = forwardPortAlgorithm - stickySessions, err := getStickySessions(service) + return getSendProxyV2(service, nodePort) +} + +func getTimeoutClient(service *v1.Service) (time.Duration, error) { + timeoutClient, ok := service.Annotations[serviceAnnotationLoadBalancerTimeoutClient] + if !ok { + return time.ParseDuration("10m") + } + + timeoutClientDuration, err := time.ParseDuration(timeoutClient) if err != nil { - return nil, err + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerTimeoutClient) + return time.Duration(0), errLoadBalancerInvalidAnnotation } - request.StickySessions = stickySessions + return timeoutClientDuration, nil +} - if stickySessions == scwlb.StickySessionsTypeCookie { - stickySessionsCookieName, err := getStickySessionsCookieName(service) - if err != nil { - return nil, err - } - if stickySessionsCookieName == "" { - klog.Errorf("missing annotation %s", serviceAnnotationLoadBalancerStickySessionsCookieName) - return nil, NewAnnorationError(serviceAnnotationLoadBalancerStickySessionsCookieName, stickySessionsCookieName) - } - request.StickySessionsCookieName = stickySessionsCookieName +func getTimeoutServer(service *v1.Service) (time.Duration, error) { + timeoutServer, ok := service.Annotations[serviceAnnotationLoadBalancerTimeoutServer] + if !ok { + return time.ParseDuration("10s") } - proxyProtocol, err := getProxyProtocol(service, nodePort) + timeoutServerDuration, err := time.ParseDuration(timeoutServer) if err != nil { - return nil, err + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerTimeoutServer) + return time.Duration(0), errLoadBalancerInvalidAnnotation } - request.ProxyProtocol = proxyProtocol - timeoutServer, err := getTimeoutServer(service) + return timeoutServerDuration, nil +} + +func getTimeoutConnect(service *v1.Service) (time.Duration, error) { + timeoutConnect, ok := service.Annotations[serviceAnnotationLoadBalancerTimeoutConnect] + if !ok { + return time.ParseDuration("10m") + } + + timeoutConnectDuration, err := time.ParseDuration(timeoutConnect) if err != nil { - return nil, err + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerTimeoutConnect) + return time.Duration(0), errLoadBalancerInvalidAnnotation } - request.TimeoutServer = &timeoutServer + return timeoutConnectDuration, nil +} - timeoutConnect, err := getTimeoutConnect(service) +func getTimeoutTunnel(service *v1.Service) (time.Duration, error) { + timeoutTunnel, ok := service.Annotations[serviceAnnotationLoadBalancerTimeoutTunnel] + if !ok { + return time.ParseDuration("10m") + } + + timeoutTunnelDuration, err := time.ParseDuration(timeoutTunnel) if err != nil { - return nil, err + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerTimeoutTunnel) + return time.Duration(0), errLoadBalancerInvalidAnnotation } - request.TimeoutConnect = &timeoutConnect + return timeoutTunnelDuration, nil +} - timeoutTunnel, err := getTimeoutTunnel(service) - if err != nil { - return nil, err +func getOnMarkedDownAction(service *v1.Service) (scwlb.OnMarkedDownAction, error) { + onMarkedDownAction, ok := service.Annotations[serviceAnnotationLoadBalancerOnMarkedDownAction] + if !ok { + return scwlb.OnMarkedDownActionOnMarkedDownActionNone, nil } - request.TimeoutTunnel = &timeoutTunnel + onMarkedDownActionValue := scwlb.OnMarkedDownAction(onMarkedDownAction) - onMarkedDownAction, err := getOnMarkedDownAction(service) - if err != nil { - return nil, err + if onMarkedDownActionValue != scwlb.OnMarkedDownActionOnMarkedDownActionNone && onMarkedDownActionValue != scwlb.OnMarkedDownActionShutdownSessions { + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerOnMarkedDownAction) + return "", errLoadBalancerInvalidAnnotation } - request.OnMarkedDownAction = onMarkedDownAction + return onMarkedDownActionValue, nil +} - healthCheck := &scwlb.HealthCheck{ - Port: nodePort, +func getRedisatchAttemptCount(service *v1.Service) (*int32, error) { + redispatchAttemptCount, ok := service.Annotations[serviceAnnotationLoadBalancerRedispatchAttemptCount] + if !ok { + var v int32 = 0 + return &v, nil } + redispatchAttemptCountInt, err := strconv.Atoi(redispatchAttemptCount) + if err != nil { + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerRedispatchAttemptCount) + return nil, errLoadBalancerInvalidAnnotation - healthCheckDelay, err := getHealthCheckDelay(service) + } + redispatchAttemptCountInt32 := int32(redispatchAttemptCountInt) + return &redispatchAttemptCountInt32, nil +} + +func getMaxRetries(service *v1.Service) (*int32, error) { + maxRetriesCount, ok := service.Annotations[serviceAnnotationLoadBalancerMaxRetries] + if !ok { + var v int32 = 3 + return &v, nil + } + maxRetriesCountInt, err := strconv.Atoi(maxRetriesCount) if err != nil { - return nil, err + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerMaxRetries) + return nil, errLoadBalancerInvalidAnnotation + } + maxRetriesCountInt32 := int32(maxRetriesCountInt) + return &maxRetriesCountInt32, nil +} - healthCheck.CheckDelay = &healthCheckDelay +func getHealthCheckDelay(service *v1.Service) (time.Duration, error) { + healthCheckDelay, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckDelay] + if !ok { + return time.ParseDuration("5s") + } - healthCheckTimeout, err := getHealthCheckTimeout(service) + healthCheckDelayDuration, err := time.ParseDuration(healthCheckDelay) if err != nil { - return nil, err + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthCheckDelay) + return time.Duration(0), errLoadBalancerInvalidAnnotation } - healthCheck.CheckTimeout = &healthCheckTimeout + return healthCheckDelayDuration, nil +} - healthCheckMaxRetries, err := getHealthCheckMaxRetries(service) - if err != nil { - return nil, err +func getHealthCheckTimeout(service *v1.Service) (time.Duration, error) { + healthCheckTimeout, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckTimeout] + if !ok { + return time.ParseDuration("5s") } - healthCheckTransientCheckDelay, err := getHealthCheckTransientCheckDelay(service) + healthCheckTimeoutDuration, err := time.ParseDuration(healthCheckTimeout) if err != nil { - return nil, err + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthCheckTimeout) + return time.Duration(0), errLoadBalancerInvalidAnnotation } - healthCheck.TransientCheckDelay = healthCheckTransientCheckDelay - healthCheck.CheckMaxRetries = healthCheckMaxRetries + return healthCheckTimeoutDuration, nil +} + +func getHealthCheckMaxRetries(service *v1.Service) (int32, error) { + healthCheckMaxRetries, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckMaxRetries] + if !ok { + return 5, nil + } - healthCheckType, err := getHealthCheckType(service, nodePort) + healthCheckMaxRetriesInt, err := strconv.Atoi(healthCheckMaxRetries) if err != nil { - return nil, err + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthCheckMaxRetries) + return 0, errLoadBalancerInvalidAnnotation } - switch healthCheckType { - case "mysql": - hc, err := getMysqlHealthCheck(service, nodePort) - if err != nil { - return nil, err - } - healthCheck.MysqlConfig = hc - case "ldap": - hc, err := getLdapHealthCheck(service, nodePort) - if err != nil { - return nil, err - } - healthCheck.LdapConfig = hc - case "redis": - hc, err := getRedisHealthCheck(service, nodePort) - if err != nil { - return nil, err - } - healthCheck.RedisConfig = hc - case "pgsql": - hc, err := getPgsqlHealthCheck(service, nodePort) - if err != nil { - return nil, err - } - healthCheck.PgsqlConfig = hc - case "tcp": - hc, err := getTCPHealthCheck(service, nodePort) - if err != nil { - return nil, err - } - healthCheck.TCPConfig = hc - case "http": - hc, err := getHTTPHealthCheck(service, nodePort) - if err != nil { - return nil, err - } - healthCheck.HTTPConfig = hc - case "https": - hc, err := getHTTPSHealthCheck(service, nodePort) - if err != nil { - return nil, err - } - healthCheck.HTTPSConfig = hc - default: - klog.Errorf("wrong value for healthCheckType") + return int32(healthCheckMaxRetriesInt), nil +} + +func getHealthCheckTransientCheckDelay(service *v1.Service) (*scw.Duration, error) { + transientCheckDelay, ok := service.Annotations[serviceAnnotationLoadBalancerHealthTransientCheckDelay] + if !ok { + return nil, nil + } + transientCheckDelayDuration, err := time.ParseDuration(transientCheckDelay) + if err != nil { + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthTransientCheckDelay) return nil, errLoadBalancerInvalidAnnotation } - request.HealthCheck = healthCheck + durationpb := durationpb.New(transientCheckDelayDuration) - return request, nil + return &scw.Duration{ + Seconds: durationpb.Seconds, + Nanos: durationpb.Nanos, + }, nil } -// createPrivateServiceStatus creates a LoadBalancer status for services with private load balancers -func (l *loadbalancers) createPrivateServiceStatus(service *v1.Service, lb *scwlb.LB) (*v1.LoadBalancerStatus, error) { - if l.pnID == "" { - return nil, fmt.Errorf("cannot make status for service %s/%s: private load balancer requires a private network", service.Namespace, service.Name) +func getForceInternalIP(service *v1.Service) bool { + forceInternalIP, ok := service.Annotations[serviceAnnotationLoadBalancerForceInternalIP] + if !ok { + return false } + value, err := strconv.ParseBool(forceInternalIP) + if err != nil { + return false + } + return value +} - region, err := lb.Zone.Region() +func getUseHostname(service *v1.Service) bool { + useHostname, ok := service.Annotations[serviceAnnotationLoadBalancerUseHostname] + if !ok { + return false + } + value, err := strconv.ParseBool(useHostname) if err != nil { - return nil, fmt.Errorf("error making status for service %s/%s: %v", service.Namespace, service.Name, err) + return false } + return value +} - status := &v1.LoadBalancerStatus{} +func getForwardProtocol(service *v1.Service, nodePort int32) (scwlb.Protocol, error) { + httpProtocol := service.Annotations[serviceAnnotationLoadBalancerProtocolHTTP] - if getUseHostname(service) { - pn, err := l.vpc.GetPrivateNetwork(&scwvpc.GetPrivateNetworkRequest{ - Region: region, - PrivateNetworkID: l.pnID, - }) - if err != nil { - return nil, fmt.Errorf("unable to query private network for lb %s: %v", lb.Name, err) + var svcPort int32 = -1 + for _, p := range service.Spec.Ports { + if p.NodePort == nodePort { + svcPort = p.Port } + } + if svcPort == -1 { + klog.Errorf("no valid port found") + return "", errLoadBalancerInvalidAnnotation + } - status.Ingress = []v1.LoadBalancerIngress{ - { - Hostname: fmt.Sprintf("%s.%s", lb.ID, pn.Name), - }, + isHTTP, err := isPortInRange(httpProtocol, svcPort) + if err != nil { + klog.Errorf("unable to check if port %d is in range %s", svcPort, httpProtocol) + return "", err + } + + if isHTTP { + return scwlb.ProtocolHTTP, nil + } + + return scwlb.ProtocolTCP, nil +} + +func getCertificateIDs(service *v1.Service, port int32) ([]string, error) { + certificates := service.Annotations[serviceAnnotationLoadBalancerCertificateIDs] + ids := []string{} + if certificates == "" { + return ids, nil + } + + for _, perPortCertificate := range strings.Split(certificates, ";") { + split := strings.Split(perPortCertificate, ":") + if len(split) == 1 { + ids = append(ids, strings.Split(split[0], ",")...) + continue } - } else { - ipamRes, err := l.ipam.ListIPs(&scwipam.ListIPsRequest{ - ProjectID: &lb.ProjectID, - ResourceType: scwipam.ResourceTypeLBServer, - ResourceID: &lb.ID, - IsIPv6: scw.BoolPtr(false), - Region: region, - }) + inRange, err := isPortInRange(split[0], port) if err != nil { - return nil, fmt.Errorf("unable to query ipam for lb %s: %v", lb.Name, err) + klog.Errorf("unable to check if port %d is in range %s", port, split[0]) + return nil, err } - - if len(ipamRes.IPs) == 0 { - return nil, fmt.Errorf("no private network ip for lb %s", lb.Name) + if inRange { + ids = append(ids, strings.Split(split[1], ",")...) } - - status.Ingress = make([]v1.LoadBalancerIngress, len(ipamRes.IPs)) - for idx, ip := range ipamRes.IPs { - status.Ingress[idx].IP = ip.Address.IP.String() + } + // normalize the ids (ie strip the region prefix if any) + for i := range ids { + if strings.Contains(ids[i], "/") { + splitID := strings.Split(ids[i], "/") + if len(splitID) != 2 { + klog.Errorf("unable to get certificate ID from %s", ids[i]) + return nil, fmt.Errorf("unable to get certificate ID from %s", ids[i]) + } + ids[i] = splitID[1] } } - return status, nil + return ids, nil } -// createPublicServiceStatus creates a LoadBalancer status for services with public load balancers -func (l *loadbalancers) createPublicServiceStatus(service *v1.Service, lb *scwlb.LB) (*v1.LoadBalancerStatus, error) { - status := &v1.LoadBalancerStatus{} - status.Ingress = make([]v1.LoadBalancerIngress, 0) - for _, ip := range lb.IP { - // Skip ipv6 entries - if i := net.ParseIP(ip.IPAddress); i.To4() == nil { - continue - } - - if getUseHostname(service) { - status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{Hostname: ip.Reverse}) - } else { - status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{IP: ip.IPAddress}) +func getValueForPort(service *v1.Service, nodePort int32, fullValue string) (string, error) { + var svcPort int32 = -1 + for _, p := range service.Spec.Ports { + if p.NodePort == nodePort { + svcPort = p.Port } } - if len(status.Ingress) == 0 { - return nil, fmt.Errorf("no ipv4 found for lb %s", lb.Name) + value := "" + + for _, perPort := range strings.Split(fullValue, ";") { + split := strings.Split(perPort, ":") + if len(split) == 1 { + if value == "" { + value = split[0] + } + continue + } + if len(split) > 2 { + return "", fmt.Errorf("annotation with value %s is wrongly formatted, should be `port1:value1;port2,port3:value2`", fullValue) + } + inRange, err := isPortInRange(split[0], svcPort) + if err != nil { + klog.Errorf("unable to check if port %d is in range %s", svcPort, split[0]) + return "", err + } + if inRange { + value = split[1] + } } - return status, nil + return value, nil } -// createServiceStatus creates a LoadBalancer status for the service -func (l *loadbalancers) createServiceStatus(service *v1.Service, lb *scwlb.LB) (*v1.LoadBalancerStatus, error) { - lbPrivate, err := svcPrivate(service) - if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerPrivate) - return nil, fmt.Errorf("invalid value for annotation %s: expected boolean", serviceAnnotationLoadBalancerPrivate) +func getHealthCheckType(service *v1.Service, nodePort int32) (string, error) { + annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckType] + if !ok { + return "tcp", nil } - if lbPrivate { - return l.createPrivateServiceStatus(service, lb) + hcValue, err := getValueForPort(service, nodePort, annotation) + if err != nil { + klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckType, nodePort) + return "", err } - return l.createPublicServiceStatus(service, lb) + return hcValue, nil +} + +func getRedisHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckRedisConfig, error) { + return &scwlb.HealthCheckRedisConfig{}, nil +} + +func getLdapHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckLdapConfig, error) { + return &scwlb.HealthCheckLdapConfig{}, nil +} + +func getTCPHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckTCPConfig, error) { + return &scwlb.HealthCheckTCPConfig{}, nil } -func getLoadBalancerID(service *v1.Service) (scw.Zone, string, error) { - annoLoadBalancerID, ok := service.Annotations[serviceAnnotationLoadBalancerID] +func getPgsqlHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckPgsqlConfig, error) { + annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckPgsqlUser] if !ok { - return "", "", errLoadBalancerInvalidAnnotation - } - - splitLoadBalancerID := strings.Split(strings.ToLower(annoLoadBalancerID), "/") - if len(splitLoadBalancerID) != 2 { - return "", "", errLoadBalancerInvalidLoadBalancerID + return nil, nil } - if validation.IsRegion(splitLoadBalancerID[0]) { - zone := splitLoadBalancerID[0] + "-1" - return scw.Zone(zone), splitLoadBalancerID[1], nil + user, err := getValueForPort(service, nodePort, annotation) + if err != nil { + klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckPgsqlUser, nodePort) + return nil, err } - return scw.Zone(splitLoadBalancerID[0]), splitLoadBalancerID[1], nil + return &scwlb.HealthCheckPgsqlConfig{ + User: user, + }, nil } -func getForwardPortAlgorithm(service *v1.Service) (scwlb.ForwardPortAlgorithm, error) { - forwardPortAlgorithm, ok := service.Annotations[serviceAnnotationLoadBalancerForwardPortAlgorithm] +func getMysqlHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckMysqlConfig, error) { + annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckMysqlUser] if !ok { - return scwlb.ForwardPortAlgorithmRoundrobin, nil + return nil, nil } - forwardPortAlgorithmValue := scwlb.ForwardPortAlgorithm(forwardPortAlgorithm) - - if forwardPortAlgorithmValue != scwlb.ForwardPortAlgorithmRoundrobin && forwardPortAlgorithmValue != scwlb.ForwardPortAlgorithmLeastconn && forwardPortAlgorithmValue != scwlb.ForwardPortAlgorithmFirst { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerForwardPortAlgorithm) - return "", errLoadBalancerInvalidAnnotation + user, err := getValueForPort(service, nodePort, annotation) + if err != nil { + klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckMysqlUser, nodePort) + return nil, err } - return forwardPortAlgorithmValue, nil + return &scwlb.HealthCheckMysqlConfig{ + User: user, + }, nil } -func getStickySessions(service *v1.Service) (scwlb.StickySessionsType, error) { - stickySessions, ok := service.Annotations[serviceAnnotationLoadBalancerStickySessions] +func getHTTPHealthCheckCode(service *v1.Service, nodePort int32) (int32, error) { + annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckHTTPCode] if !ok { - return scwlb.StickySessionsTypeNone, nil + return 200, nil } - stickySessionsValue := scwlb.StickySessionsType(stickySessions) - - if stickySessionsValue != scwlb.StickySessionsTypeNone && stickySessionsValue != scwlb.StickySessionsTypeCookie && stickySessionsValue != scwlb.StickySessionsTypeTable { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerStickySessions) - return "", errLoadBalancerInvalidAnnotation + stringCode, err := getValueForPort(service, nodePort, annotation) + if err != nil { + klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckHTTPCode, nodePort) + return 0, err } - return stickySessionsValue, nil -} - -func getStickySessionsCookieName(service *v1.Service) (string, error) { - stickySessionsCookieName, ok := service.Annotations[serviceAnnotationLoadBalancerStickySessionsCookieName] - if !ok { - return "", nil + code, err := strconv.Atoi(stringCode) + if err != nil { + klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthCheckHTTPCode) + return 0, errLoadBalancerInvalidAnnotation } - return stickySessionsCookieName, nil + return int32(code), nil } -func getSendProxyV2(service *v1.Service, nodePort int32) (scwlb.ProxyProtocol, error) { - sendProxyV2, ok := service.Annotations[serviceAnnotationLoadBalancerSendProxyV2] +func getHTTPHealthCheckURI(service *v1.Service, nodePort int32) (string, error) { + annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckHTTPURI] if !ok { - return scwlb.ProxyProtocolProxyProtocolNone, nil + return "/", nil } - sendProxyV2Value, err := strconv.ParseBool(sendProxyV2) + uri, err := getValueForPort(service, nodePort, annotation) if err != nil { - var svcPort int32 = -1 - for _, p := range service.Spec.Ports { - if p.NodePort == nodePort { - svcPort = p.Port - } - } - if svcPort == -1 { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerSendProxyV2) - return "", errLoadBalancerInvalidAnnotation - } - - ports := strings.Split(strings.ReplaceAll(sendProxyV2, " ", ""), ",") - for _, port := range ports { - intPort, err := strconv.ParseInt(port, 0, 64) - if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerSendProxyV2) - return "", errLoadBalancerInvalidAnnotation - } - if int64(svcPort) == intPort { - return scwlb.ProxyProtocolProxyProtocolV2, nil - } - } - return scwlb.ProxyProtocolProxyProtocolNone, nil - } - - if sendProxyV2Value { - return scwlb.ProxyProtocolProxyProtocolV2, nil + klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckHTTPURI, nodePort) + return "", err } - return scwlb.ProxyProtocolProxyProtocolNone, nil + return uri, nil } -func isPortInRange(r string, p int32) (bool, error) { - boolValue, err := strconv.ParseBool(r) - if err == nil && r != "1" && r != "0" { - return boolValue, nil - } - if r == "*" { - return true, nil - } - if r == "" { - return false, nil - } - ports := strings.Split(strings.ReplaceAll(r, " ", ""), ",") - for _, port := range ports { - intPort, err := strconv.ParseInt(port, 0, 64) - if err != nil { - return false, err - } - if int64(p) == intPort { - return true, nil - } +func getHTTPHealthCheckMethod(service *v1.Service, nodePort int32) (string, error) { + annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckHTTPMethod] + if !ok { + return "GET", nil } - return false, nil -} -func getLoadBalancerType(service *v1.Service) string { - return strings.ToLower(service.Annotations[serviceAnnotationLoadBalancerType]) -} + method, err := getValueForPort(service, nodePort, annotation) + if err != nil { + klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckHTTPMethod, nodePort) + return "", err + } -func getLoadBalancerZone(service *v1.Service) scw.Zone { - return scw.Zone(strings.ToLower(service.Annotations[serviceAnnotationLoadBalancerZone])) + return method, nil } -func getProxyProtocol(service *v1.Service, nodePort int32) (scwlb.ProxyProtocol, error) { - proxyProtocolV1 := service.Annotations[serviceAnnotationLoadBalancerProxyProtocolV1] - proxyProtocolV2 := service.Annotations[serviceAnnotationLoadBalancerProxyProtocolV2] - - var svcPort int32 = -1 - for _, p := range service.Spec.Ports { - if p.NodePort == nodePort { - svcPort = p.Port - } - } - if svcPort == -1 { - klog.Errorf("no valid port found") - return "", errLoadBalancerInvalidAnnotation +func getHTTPHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckHTTPConfig, error) { + code, err := getHTTPHealthCheckCode(service, nodePort) + if err != nil { + return nil, err } - isV1, err := isPortInRange(proxyProtocolV1, svcPort) + uriStr, err := getHTTPHealthCheckURI(service, nodePort) if err != nil { - klog.Errorf("unable to check if port %d is in range %s", svcPort, proxyProtocolV1) - return "", err + return nil, err } - isV2, err := isPortInRange(proxyProtocolV2, svcPort) + uri, err := url.Parse(fmt.Sprintf("http://%s", uriStr)) if err != nil { - klog.Errorf("unable to check if port %d is in range %s", svcPort, proxyProtocolV2) - return "", err + return nil, err } - - if isV1 && isV2 { - klog.Errorf("port %d is in both v1 and v2 proxy protocols", svcPort) - return "", fmt.Errorf("port %d is in both v1 and v2 proxy protocols", svcPort) + if uri.Path == "" { + uri.Path = "/" } - if isV1 { - return scwlb.ProxyProtocolProxyProtocolV1, nil - } - if isV2 { - return scwlb.ProxyProtocolProxyProtocolV2, nil + method, err := getHTTPHealthCheckMethod(service, nodePort) + if err != nil { + return nil, err } - return getSendProxyV2(service, nodePort) + return &scwlb.HealthCheckHTTPConfig{ + Method: method, + Code: &code, + URI: uri.RequestURI(), + HostHeader: uri.Host, + }, nil } -func getTimeoutClient(service *v1.Service) (time.Duration, error) { - timeoutClient, ok := service.Annotations[serviceAnnotationLoadBalancerTimeoutClient] - if !ok { - return time.ParseDuration("10m") +func getHTTPSHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckHTTPSConfig, error) { + code, err := getHTTPHealthCheckCode(service, nodePort) + if err != nil { + return nil, err } - timeoutClientDuration, err := time.ParseDuration(timeoutClient) + uriStr, err := getHTTPHealthCheckURI(service, nodePort) if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerTimeoutClient) - return time.Duration(0), errLoadBalancerInvalidAnnotation + return nil, err } - - return timeoutClientDuration, nil -} - -func getTimeoutServer(service *v1.Service) (time.Duration, error) { - timeoutServer, ok := service.Annotations[serviceAnnotationLoadBalancerTimeoutServer] - if !ok { - return time.ParseDuration("10s") + uri, err := url.Parse(fmt.Sprintf("https://%s", uriStr)) + if err != nil { + return nil, err + } + if uri.Path == "" { + uri.Path = "/" } - timeoutServerDuration, err := time.ParseDuration(timeoutServer) + method, err := getHTTPHealthCheckMethod(service, nodePort) if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerTimeoutServer) - return time.Duration(0), errLoadBalancerInvalidAnnotation + return nil, err } - return timeoutServerDuration, nil + return &scwlb.HealthCheckHTTPSConfig{ + Method: method, + Code: &code, + URI: uri.Path, + HostHeader: uri.Host, + Sni: uri.Host, + }, nil } -func getTimeoutConnect(service *v1.Service) (time.Duration, error) { - timeoutConnect, ok := service.Annotations[serviceAnnotationLoadBalancerTimeoutConnect] +func svcPrivate(service *v1.Service) (bool, error) { + isPrivate, ok := service.Annotations[serviceAnnotationLoadBalancerPrivate] if !ok { - return time.ParseDuration("10m") + return false, nil } + return strconv.ParseBool(isPrivate) +} - timeoutConnectDuration, err := time.ParseDuration(timeoutConnect) - if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerTimeoutConnect) - return time.Duration(0), errLoadBalancerInvalidAnnotation - } +// Original version: https://github.com/kubernetes/legacy-cloud-providers/blob/1aa918bf227e52af6f8feb3fa065dabff251a0a3/aws/aws_loadbalancer.go#L117 +func getKeyValueFromAnnotation(annotation string) map[string]string { + additionalTags := make(map[string]string) + additionalTagsList := strings.TrimSpace(annotation) - return timeoutConnectDuration, nil -} + // Break up list of "Key1=Val,Key2=Val2" + tagList := strings.Split(additionalTagsList, ",") -func getTimeoutTunnel(service *v1.Service) (time.Duration, error) { - timeoutTunnel, ok := service.Annotations[serviceAnnotationLoadBalancerTimeoutTunnel] - if !ok { - return time.ParseDuration("10m") - } + // Break up "Key=Val" + for _, tagSet := range tagList { + tag := strings.Split(strings.TrimSpace(tagSet), "=") - timeoutTunnelDuration, err := time.ParseDuration(timeoutTunnel) - if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerTimeoutTunnel) - return time.Duration(0), errLoadBalancerInvalidAnnotation + // Accept "Key=val" or "Key=" or just "Key" + if len(tag) >= 2 && len(tag[0]) != 0 { + // There is a key and a value, so save it + additionalTags[tag[0]] = tag[1] + } else if len(tag) == 1 && len(tag[0]) != 0 { + // Just "Key" + additionalTags[tag[0]] = "" + } } - return timeoutTunnelDuration, nil + return additionalTags } -func getOnMarkedDownAction(service *v1.Service) (scwlb.OnMarkedDownAction, error) { - onMarkedDownAction, ok := service.Annotations[serviceAnnotationLoadBalancerOnMarkedDownAction] +// Original version: https://github.com/kubernetes/legacy-cloud-providers/blob/1aa918bf227e52af6f8feb3fa065dabff251a0a3/aws/aws_loadbalancer.go#L1631 +func filterNodes(service *v1.Service, nodes []*v1.Node) []*v1.Node { + nodeLabels, ok := service.Annotations[serviceAnnotationLoadBalancerTargetNodeLabels] if !ok { - return scwlb.OnMarkedDownActionOnMarkedDownActionNone, nil + return nodes } - onMarkedDownActionValue := scwlb.OnMarkedDownAction(onMarkedDownAction) + targetNodeLabels := getKeyValueFromAnnotation(nodeLabels) - if onMarkedDownActionValue != scwlb.OnMarkedDownActionOnMarkedDownActionNone && onMarkedDownActionValue != scwlb.OnMarkedDownActionShutdownSessions { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerOnMarkedDownAction) - return "", errLoadBalancerInvalidAnnotation + if len(targetNodeLabels) == 0 { + return nodes } - return onMarkedDownActionValue, nil -} + targetNodes := make([]*v1.Node, 0, len(nodes)) -func getRedisatchAttemptCount(service *v1.Service) (*int32, error) { - redispatchAttemptCount, ok := service.Annotations[serviceAnnotationLoadBalancerRedispatchAttemptCount] - if !ok { - return nil, nil - } - redispatchAttemptCountInt, err := strconv.Atoi(redispatchAttemptCount) - if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerRedispatchAttemptCount) - return nil, errLoadBalancerInvalidAnnotation + for _, node := range nodes { + if node.Labels != nil && len(node.Labels) > 0 { + allFiltersMatch := true - } - redispatchAttemptCountInt32 := int32(redispatchAttemptCountInt) - return &redispatchAttemptCountInt32, nil -} + for targetLabelKey, targetLabelValue := range targetNodeLabels { + if nodeLabelValue, ok := node.Labels[targetLabelKey]; !ok || (nodeLabelValue != targetLabelValue && targetLabelValue != "") { + allFiltersMatch = false + break + } + } -func getMaxRetries(service *v1.Service) (*int32, error) { - maxRetriesCount, ok := service.Annotations[serviceAnnotationLoadBalancerMaxRetries] - if !ok { - return nil, nil + if allFiltersMatch { + targetNodes = append(targetNodes, node) + } + } } - maxRetriesCountInt, err := strconv.Atoi(maxRetriesCount) - if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerMaxRetries) - return nil, errLoadBalancerInvalidAnnotation - } - maxRetriesCountInt32 := int32(maxRetriesCountInt) - return &maxRetriesCountInt32, nil + return targetNodes } -func getHealthCheckDelay(service *v1.Service) (time.Duration, error) { - healthCheckDelay, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckDelay] - if !ok { - return time.ParseDuration("5s") +func servicePortToFrontend(service *v1.Service, loadbalancer *scwlb.LB, port v1.ServicePort) (*scwlb.Frontend, error) { + timeoutClient, err := getTimeoutClient(service) + if err != nil { + return nil, fmt.Errorf("error getting %s annotation for loadbalancer %s: %v", + serviceAnnotationLoadBalancerTimeoutClient, loadbalancer.ID, err) } - healthCheckDelayDuration, err := time.ParseDuration(healthCheckDelay) + certificateIDs, err := getCertificateIDs(service, port.Port) if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthCheckDelay) - return time.Duration(0), errLoadBalancerInvalidAnnotation + return nil, fmt.Errorf("error getting certificate IDs for loadbalancer %s: %v", loadbalancer.ID, err) } - return healthCheckDelayDuration, nil + return &scwlb.Frontend{ + Name: fmt.Sprintf("%s_tcp_%d", string(service.UID), port.Port), + InboundPort: port.Port, + TimeoutClient: &timeoutClient, + CertificateIDs: certificateIDs, + }, nil } -func getHealthCheckTimeout(service *v1.Service) (time.Duration, error) { - healthCheckTimeout, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckTimeout] - if !ok { - return time.ParseDuration("5s") +func servicePortToBackend(service *v1.Service, loadbalancer *scwlb.LB, port v1.ServicePort, nodeIPs []string) (*scwlb.Backend, error) { + protocol, err := getForwardProtocol(service, port.NodePort) + if err != nil { + return nil, err } - healthCheckTimeoutDuration, err := time.ParseDuration(healthCheckTimeout) + forwardPortAlgorithm, err := getForwardPortAlgorithm(service) if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthCheckTimeout) - return time.Duration(0), errLoadBalancerInvalidAnnotation + return nil, err } - return healthCheckTimeoutDuration, nil -} - -func getHealthCheckMaxRetries(service *v1.Service) (int32, error) { - healthCheckMaxRetries, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckMaxRetries] - if !ok { - return 5, nil + stickySessions, err := getStickySessions(service) + if err != nil { + return nil, err } - healthCheckMaxRetriesInt, err := strconv.Atoi(healthCheckMaxRetries) + proxyProtocol, err := getProxyProtocol(service, port.NodePort) if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthCheckMaxRetries) - return 0, errLoadBalancerInvalidAnnotation + return nil, err } - return int32(healthCheckMaxRetriesInt), nil -} - -func getHealthCheckTransientCheckDelay(service *v1.Service) (*scw.Duration, error) { - transientCheckDelay, ok := service.Annotations[serviceAnnotationLoadBalancerHealthTransientCheckDelay] - if !ok { - return nil, nil - } - transientCheckDelayDuration, err := time.ParseDuration(transientCheckDelay) + timeoutServer, err := getTimeoutServer(service) if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthTransientCheckDelay) - return nil, errLoadBalancerInvalidAnnotation + return nil, err } - durationpb := durationpb.New(transientCheckDelayDuration) - - return &scw.Duration{ - Seconds: durationpb.Seconds, - Nanos: durationpb.Nanos, - }, nil -} - -func getForceInternalIP(service *v1.Service) bool { - forceInternalIP, ok := service.Annotations[serviceAnnotationLoadBalancerForceInternalIP] - if !ok { - return false - } - value, err := strconv.ParseBool(forceInternalIP) + timeoutConnect, err := getTimeoutConnect(service) if err != nil { - return false + return nil, err } - return value -} -func getUseHostname(service *v1.Service) bool { - useHostname, ok := service.Annotations[serviceAnnotationLoadBalancerUseHostname] - if !ok { - return false + timeoutTunnel, err := getTimeoutTunnel(service) + if err != nil { + return nil, err } - value, err := strconv.ParseBool(useHostname) + + onMarkedDownAction, err := getOnMarkedDownAction(service) if err != nil { - return false + return nil, err } - return value -} -func getForwardProtocol(service *v1.Service, nodePort int32) (scwlb.Protocol, error) { - httpProtocol := service.Annotations[serviceAnnotationLoadBalancerProtocolHTTP] + redispatchAttemptCount, err := getRedisatchAttemptCount(service) + if err != nil { + return nil, err + } - var svcPort int32 = -1 - for _, p := range service.Spec.Ports { - if p.NodePort == nodePort { - svcPort = p.Port - } + maxRetries, err := getMaxRetries(service) + if err != nil { + return nil, err } - if svcPort == -1 { - klog.Errorf("no valid port found") - return "", errLoadBalancerInvalidAnnotation + + healthCheck := &scwlb.HealthCheck{ + Port: port.NodePort, } - isHTTP, err := isPortInRange(httpProtocol, svcPort) + healthCheckDelay, err := getHealthCheckDelay(service) if err != nil { - klog.Errorf("unable to check if port %d is in range %s", svcPort, httpProtocol) - return "", err + return nil, err } + healthCheck.CheckDelay = &healthCheckDelay - if isHTTP { - return scwlb.ProtocolHTTP, nil + healthCheckTimeout, err := getHealthCheckTimeout(service) + if err != nil { + return nil, err } + healthCheck.CheckTimeout = &healthCheckTimeout - return scwlb.ProtocolTCP, nil -} + healthCheckMaxRetries, err := getHealthCheckMaxRetries(service) + if err != nil { + return nil, err + } + healthCheck.CheckMaxRetries = healthCheckMaxRetries -func getCertificateIDs(service *v1.Service, port int32) ([]string, error) { - certificates := service.Annotations[serviceAnnotationLoadBalancerCertificateIDs] - if certificates == "" { - return nil, nil + healthCheckTransientCheckDelay, err := getHealthCheckTransientCheckDelay(service) + if err != nil { + return nil, err } + healthCheck.TransientCheckDelay = healthCheckTransientCheckDelay - ids := []string{} + healthCheckType, err := getHealthCheckType(service, port.NodePort) + if err != nil { + return nil, err + } - for _, perPortCertificate := range strings.Split(certificates, ";") { - split := strings.Split(perPortCertificate, ":") - if len(split) == 1 { - ids = append(ids, strings.Split(split[0], ",")...) - continue + switch healthCheckType { + case "mysql": + hc, err := getMysqlHealthCheck(service, port.NodePort) + if err != nil { + return nil, err } - inRange, err := isPortInRange(split[0], port) + healthCheck.MysqlConfig = hc + case "ldap": + hc, err := getLdapHealthCheck(service, port.NodePort) if err != nil { - klog.Errorf("unable to check if port %d is in range %s", port, split[0]) return nil, err } - if inRange { - ids = append(ids, strings.Split(split[1], ",")...) + healthCheck.LdapConfig = hc + case "redis": + hc, err := getRedisHealthCheck(service, port.NodePort) + if err != nil { + return nil, err } - } - // normalize the ids (ie strip the region prefix if any) - for i := range ids { - if strings.Contains(ids[i], "/") { - splitID := strings.Split(ids[i], "/") - if len(splitID) != 2 { - klog.Errorf("unable to get certificate ID from %s", ids[i]) - return nil, fmt.Errorf("unable to get certificate ID from %s", ids[i]) - } - ids[i] = splitID[1] + healthCheck.RedisConfig = hc + case "pgsql": + hc, err := getPgsqlHealthCheck(service, port.NodePort) + if err != nil { + return nil, err } - } - - return ids, nil -} - -func getValueForPort(service *v1.Service, nodePort int32, fullValue string) (string, error) { - var svcPort int32 = -1 - for _, p := range service.Spec.Ports { - if p.NodePort == nodePort { - svcPort = p.Port + healthCheck.PgsqlConfig = hc + case "tcp": + hc, err := getTCPHealthCheck(service, port.NodePort) + if err != nil { + return nil, err } - } - - value := "" - - for _, perPort := range strings.Split(fullValue, ";") { - split := strings.Split(perPort, ":") - if len(split) == 1 { - if value == "" { - value = split[0] - } - continue + healthCheck.TCPConfig = hc + case "http": + hc, err := getHTTPHealthCheck(service, port.NodePort) + if err != nil { + return nil, err } - if len(split) > 2 { - return "", fmt.Errorf("annotation with value %s is wrongly formatted, should be `port1:value1;port2,port3:value2`", fullValue) + healthCheck.HTTPConfig = hc + case "https": + hc, err := getHTTPSHealthCheck(service, port.NodePort) + if err != nil { + return nil, err } - inRange, err := isPortInRange(split[0], svcPort) + healthCheck.HTTPSConfig = hc + default: + klog.Errorf("wrong value for healthCheckType") + return nil, errLoadBalancerInvalidAnnotation + } + + backend := &scwlb.Backend{ + Name: fmt.Sprintf("%s_tcp_%d", string(service.UID), port.NodePort), + Pool: nodeIPs, + ForwardPort: port.NodePort, + ForwardProtocol: protocol, + ForwardPortAlgorithm: forwardPortAlgorithm, + StickySessions: stickySessions, + ProxyProtocol: proxyProtocol, + TimeoutServer: &timeoutServer, + TimeoutConnect: &timeoutConnect, + TimeoutTunnel: &timeoutTunnel, + OnMarkedDownAction: onMarkedDownAction, + HealthCheck: healthCheck, + RedispatchAttemptCount: redispatchAttemptCount, + MaxRetries: maxRetries, + } + + if stickySessions == scwlb.StickySessionsTypeCookie { + stickySessionsCookieName, err := getStickySessionsCookieName(service) if err != nil { - klog.Errorf("unable to check if port %d is in range %s", svcPort, split[0]) - return "", err + return nil, err } - if inRange { - value = split[1] + if stickySessionsCookieName == "" { + klog.Errorf("missing annotation %s", serviceAnnotationLoadBalancerStickySessionsCookieName) + return nil, NewAnnorationError(serviceAnnotationLoadBalancerStickySessionsCookieName, stickySessionsCookieName) } + backend.StickySessionsCookieName = stickySessionsCookieName } - return value, nil + return backend, nil } -func getHealthCheckType(service *v1.Service, nodePort int32) (string, error) { - annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckType] - if !ok { - return "tcp", nil - } +func serviceToLB(service *v1.Service, loadbalancer *scwlb.LB, nodeIPs []string) (map[int32]*scwlb.Frontend, map[int32]*scwlb.Backend, error) { + frontends := map[int32]*scwlb.Frontend{} + backends := map[int32]*scwlb.Backend{} - hcValue, err := getValueForPort(service, nodePort, annotation) - if err != nil { - klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckType, nodePort) - return "", err - } + for _, port := range service.Spec.Ports { + frontend, err := servicePortToFrontend(service, loadbalancer, port) + if err != nil { + return nil, nil, fmt.Errorf("failed to prepare frontend for port %d: %v", port.Port, err) + } - return hcValue, nil -} + backend, err := servicePortToBackend(service, loadbalancer, port, nodeIPs) + if err != nil { + return nil, nil, fmt.Errorf("failed to prepare backend for port %d: %v", port.Port, err) + } -func getRedisHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckRedisConfig, error) { - return &scwlb.HealthCheckRedisConfig{}, nil -} + frontends[port.Port] = frontend + backends[port.NodePort] = backend + } -func getLdapHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckLdapConfig, error) { - return &scwlb.HealthCheckLdapConfig{}, nil + return frontends, backends, nil } -func getTCPHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckTCPConfig, error) { - return &scwlb.HealthCheckTCPConfig{}, nil -} +func frontendEquals(got, want *scwlb.Frontend) bool { + if got == nil || want == nil { + return got == want + } -func getPgsqlHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckPgsqlConfig, error) { - annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckPgsqlUser] - if !ok { - return nil, nil + if got.Name != want.Name { + klog.V(3).Infof("frontend.Name: %s - %s", got.Name, want.Name) + return false + } + if got.InboundPort != want.InboundPort { + klog.V(3).Infof("frontend.InboundPort: %s - %s", got.InboundPort, want.InboundPort) + return false + } + if !durationPtrEqual(got.TimeoutClient, want.TimeoutClient) { + klog.V(3).Infof("frontend.TimeoutClient: %s - %s", got.TimeoutClient, want.TimeoutClient) + return false } - user, err := getValueForPort(service, nodePort, annotation) - if err != nil { - klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckPgsqlUser, nodePort) - return nil, err + if !stringArrayEqual(got.CertificateIDs, want.CertificateIDs) { + klog.V(3).Infof("frontend.CertificateIDs: %s - %s", got.CertificateIDs, want.CertificateIDs) + return false } - return &scwlb.HealthCheckPgsqlConfig{ - User: user, - }, nil + return true } -func getMysqlHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckMysqlConfig, error) { - annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckMysqlUser] - if !ok { - return nil, nil +func backendEquals(got, want *scwlb.Backend) bool { + if got == nil || want == nil { + return got == want } - user, err := getValueForPort(service, nodePort, annotation) - if err != nil { - klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckMysqlUser, nodePort) - return nil, err + if got.Name != want.Name { + klog.V(3).Infof("backend.Name: %s - %s", got.Name, want.Name) + return false + } + if got.ForwardPort != want.ForwardPort { + klog.V(3).Infof("backend.ForwardPort: %s - %s", got.ForwardPort, want.ForwardPort) + return false + } + if got.ForwardProtocol != want.ForwardProtocol { + klog.V(3).Infof("backend.ForwardProtocol: %s - %s", got.ForwardProtocol, want.ForwardProtocol) + return false + } + if got.ForwardPortAlgorithm != want.ForwardPortAlgorithm { + klog.V(3).Infof("backend.ForwardPortAlgorithm: %s - %s", got.ForwardPortAlgorithm, want.ForwardPortAlgorithm) + return false + } + if got.StickySessions != want.StickySessions { + klog.V(3).Infof("backend.StickySessions: %s - %s", got.StickySessions, want.StickySessions) + return false + } + if got.ProxyProtocol != want.ProxyProtocol { + klog.V(3).Infof("backend.ProxyProtocol: %s - %s", got.ProxyProtocol, want.ProxyProtocol) + return false + } + if !durationPtrEqual(got.TimeoutServer, want.TimeoutServer) { + klog.V(3).Infof("backend.TimeoutServer: %s - %s", got.TimeoutServer, want.TimeoutServer) + return false + } + if !durationPtrEqual(got.TimeoutConnect, want.TimeoutConnect) { + klog.V(3).Infof("backend.TimeoutConnect: %s - %s", got.TimeoutConnect, want.TimeoutConnect) + return false + } + if !durationPtrEqual(got.TimeoutTunnel, want.TimeoutTunnel) { + klog.V(3).Infof("backend.TimeoutTunnel: %s - %s", got.TimeoutTunnel, want.TimeoutTunnel) + return false + } + if got.OnMarkedDownAction != want.OnMarkedDownAction { + klog.V(3).Infof("backend.OnMarkedDownAction: %s - %s", got.OnMarkedDownAction, want.OnMarkedDownAction) + return false + } + if !int32PtrEqual(got.RedispatchAttemptCount, want.RedispatchAttemptCount) { + klog.V(3).Infof("backend.RedispatchAttemptCount: %s - %s", got.RedispatchAttemptCount, want.RedispatchAttemptCount) + return false + } + if !int32PtrEqual(got.MaxRetries, want.MaxRetries) { + klog.V(3).Infof("backend.MaxRetries: %s - %s", got.MaxRetries, want.MaxRetries) + return false + } + if got.StickySessionsCookieName != want.StickySessionsCookieName { + klog.V(3).Infof("backend.StickySessionsCookieName: %s - %s", got.StickySessionsCookieName, want.StickySessionsCookieName) + return false } - return &scwlb.HealthCheckMysqlConfig{ - User: user, - }, nil + if !reflect.DeepEqual(got.HealthCheck, want.HealthCheck) { + klog.V(3).Infof("backend.HealthCheck: %s - %s", got.HealthCheck, want.HealthCheck) + return false + } + + return true } -func getHTTPHealthCheckCode(service *v1.Service, nodePort int32) (int32, error) { - annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckHTTPCode] - if !ok { - return 200, nil +type frontendOps struct { + remove map[int32]*scwlb.Frontend + update map[int32]*scwlb.Frontend + create map[int32]*scwlb.Frontend + keep map[int32]*scwlb.Frontend +} + +func compareFrontends(got []*scwlb.Frontend, want map[int32]*scwlb.Frontend) frontendOps { + remove := make(map[int32]*scwlb.Frontend) + update := make(map[int32]*scwlb.Frontend) + create := make(map[int32]*scwlb.Frontend) + keep := make(map[int32]*scwlb.Frontend) + + // Check for deletions and updates + for _, current := range got { + if target, ok := want[current.InboundPort]; ok { + if !frontendEquals(current, target) { + target.ID = current.ID + update[target.InboundPort] = target + } else { + keep[target.InboundPort] = current + } + } else { + remove[current.InboundPort] = current + } } - stringCode, err := getValueForPort(service, nodePort, annotation) - if err != nil { - klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckHTTPCode, nodePort) - return 0, err + // Check for additions + for _, target := range want { + found := false + for _, current := range got { + if current.InboundPort == target.InboundPort { + found = true + break + } + } + if !found { + create[target.InboundPort] = target + } } - code, err := strconv.Atoi(stringCode) - if err != nil { - klog.Errorf("invalid value for annotation %s", serviceAnnotationLoadBalancerHealthCheckHTTPCode) - return 0, errLoadBalancerInvalidAnnotation + return frontendOps{ + remove: remove, + update: update, + create: create, + keep: keep, } +} - return int32(code), nil +type backendOps struct { + remove map[int32]*scwlb.Backend + update map[int32]*scwlb.Backend + create map[int32]*scwlb.Backend + keep map[int32]*scwlb.Backend } -func getHTTPHealthCheckURI(service *v1.Service, nodePort int32) (string, error) { - annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckHTTPURI] - if !ok { - return "/", nil +func compareBackends(got []*scwlb.Backend, want map[int32]*scwlb.Backend) backendOps { + remove := make(map[int32]*scwlb.Backend) + update := make(map[int32]*scwlb.Backend) + create := make(map[int32]*scwlb.Backend) + keep := make(map[int32]*scwlb.Backend) + + // Check for deletions and updates + for _, current := range got { + if target, ok := want[current.ForwardPort]; ok { + if !backendEquals(current, target) { + target.ID = current.ID + update[target.ForwardPort] = target + } else { + keep[target.ForwardPort] = current + } + } else { + remove[current.ForwardPort] = current + } } - uri, err := getValueForPort(service, nodePort, annotation) - if err != nil { - klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckHTTPURI, nodePort) - return "", err + // Check for additions + for _, target := range want { + found := false + for _, current := range got { + if current.ForwardPort == target.ForwardPort { + found = true + break + } + } + if !found { + create[target.ForwardPort] = target + } } - return uri, nil + return backendOps{ + remove: remove, + update: update, + create: create, + keep: keep, + } } -func getHTTPHealthCheckMethod(service *v1.Service, nodePort int32) (string, error) { - annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckHTTPMethod] - if !ok { - return "GET", nil +func aclsEquals(got []*scwlb.ACL, want []*scwlb.ACLSpec) bool { + if len(got) != len(want) { + return false } - method, err := getValueForPort(service, nodePort, annotation) - if err != nil { - klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckHTTPMethod, nodePort) - return "", err + slices.SortStableFunc(got, func(a, b *scwlb.ACL) bool { return a.Index < b.Index }) + slices.SortStableFunc(want, func(a, b *scwlb.ACLSpec) bool { return a.Index < b.Index }) + for idx := range want { + if want[idx].Name != got[idx].Name { + return false + } + if want[idx].Index != got[idx].Index { + return false + } + if (want[idx].Action == nil) != (got[idx].Action == nil) { + return false + } + if want[idx].Action != nil && want[idx].Action.Type != got[idx].Action.Type { + return false + } + if (want[idx].Match == nil) != (got[idx].Match == nil) { + return false + } + if want[idx].Match != nil && !stringPtrArrayEqual(want[idx].Match.IPSubnet, got[idx].Match.IPSubnet) { + return false + } + if want[idx].Match != nil && want[idx].Match.Invert != got[idx].Match.Invert { + return false + } } - return method, nil + return true } -func getHTTPHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckHTTPConfig, error) { - code, err := getHTTPHealthCheckCode(service, nodePort) - if err != nil { - return nil, err - } - uri, err := getHTTPHealthCheckURI(service, nodePort) - if err != nil { - return nil, err - } - method, err := getHTTPHealthCheckMethod(service, nodePort) +func (l *loadbalancers) createBackend(service *v1.Service, loadbalancer *scwlb.LB, backend *scwlb.Backend) (*scwlb.Backend, error) { + b, err := l.api.CreateBackend(&scwlb.ZonedAPICreateBackendRequest{ + Zone: loadbalancer.Zone, + LBID: loadbalancer.ID, + Name: backend.Name, + ForwardProtocol: backend.ForwardProtocol, + ForwardPort: backend.ForwardPort, + ForwardPortAlgorithm: backend.ForwardPortAlgorithm, + StickySessions: backend.StickySessions, + StickySessionsCookieName: backend.StickySessionsCookieName, + HealthCheck: backend.HealthCheck, + ServerIP: backend.Pool, + TimeoutServer: backend.TimeoutServer, + TimeoutConnect: backend.TimeoutConnect, + TimeoutTunnel: backend.TimeoutTunnel, + OnMarkedDownAction: backend.OnMarkedDownAction, + ProxyProtocol: backend.ProxyProtocol, + RedispatchAttemptCount: backend.RedispatchAttemptCount, + MaxRetries: backend.MaxRetries, + }) if err != nil { return nil, err } - return &scwlb.HealthCheckHTTPConfig{ - Method: method, - Code: &code, - URI: uri, - }, nil + return b, nil } -func getHTTPSHealthCheck(service *v1.Service, nodePort int32) (*scwlb.HealthCheckHTTPSConfig, error) { - code, err := getHTTPHealthCheckCode(service, nodePort) - if err != nil { - return nil, err - } - uri, err := getHTTPHealthCheckURI(service, nodePort) - if err != nil { - return nil, err - } - method, err := getHTTPHealthCheckMethod(service, nodePort) +func (l *loadbalancers) updateBackend(service *v1.Service, loadbalancer *scwlb.LB, backend *scwlb.Backend) (*scwlb.Backend, error) { + b, err := l.api.UpdateBackend(&scwlb.ZonedAPIUpdateBackendRequest{ + Zone: loadbalancer.Zone, + BackendID: backend.ID, + Name: backend.Name, + ForwardProtocol: backend.ForwardProtocol, + ForwardPort: backend.ForwardPort, + ForwardPortAlgorithm: backend.ForwardPortAlgorithm, + StickySessions: backend.StickySessions, + StickySessionsCookieName: backend.StickySessionsCookieName, + TimeoutServer: backend.TimeoutServer, + TimeoutConnect: backend.TimeoutConnect, + TimeoutTunnel: backend.TimeoutTunnel, + OnMarkedDownAction: backend.OnMarkedDownAction, + ProxyProtocol: backend.ProxyProtocol, + RedispatchAttemptCount: backend.RedispatchAttemptCount, + MaxRetries: backend.MaxRetries, + }) if err != nil { return nil, err } - return &scwlb.HealthCheckHTTPSConfig{ - Method: method, - Code: &code, - URI: uri, - }, nil + if _, err := l.api.UpdateHealthCheck(&scwlb.ZonedAPIUpdateHealthCheckRequest{ + Zone: loadbalancer.Zone, + BackendID: backend.ID, + Port: backend.ForwardPort, + CheckDelay: backend.HealthCheck.CheckDelay, + CheckTimeout: backend.HealthCheck.CheckTimeout, + CheckMaxRetries: backend.HealthCheck.CheckMaxRetries, + CheckSendProxy: backend.HealthCheck.CheckSendProxy, + TCPConfig: backend.HealthCheck.TCPConfig, + MysqlConfig: backend.HealthCheck.MysqlConfig, + PgsqlConfig: backend.HealthCheck.PgsqlConfig, + LdapConfig: backend.HealthCheck.LdapConfig, + RedisConfig: backend.HealthCheck.RedisConfig, + HTTPConfig: backend.HealthCheck.HTTPConfig, + HTTPSConfig: backend.HealthCheck.HTTPSConfig, + TransientCheckDelay: backend.HealthCheck.TransientCheckDelay, + }); err != nil { + return nil, fmt.Errorf("failed to update healthcheck: %v", err) + } + + return b, nil } -func svcPrivate(service *v1.Service) (bool, error) { - isPrivate, ok := service.Annotations[serviceAnnotationLoadBalancerPrivate] - if !ok { - return false, nil +func (l *loadbalancers) createFrontend(service *v1.Service, loadbalancer *scwlb.LB, frontend *scwlb.Frontend, backend *scwlb.Backend) (*scwlb.Frontend, error) { + f, err := l.api.CreateFrontend(&scwlb.ZonedAPICreateFrontendRequest{ + Zone: loadbalancer.Zone, + LBID: loadbalancer.ID, + Name: frontend.Name, + InboundPort: frontend.InboundPort, + BackendID: backend.ID, + TimeoutClient: frontend.TimeoutClient, + CertificateIDs: &frontend.CertificateIDs, + EnableHTTP3: frontend.EnableHTTP3, + }) + + return f, err +} + +func (l *loadbalancers) updateFrontend(service *v1.Service, loadbalancer *scwlb.LB, frontend *scwlb.Frontend, backend *scwlb.Backend) (*scwlb.Frontend, error) { + f, err := l.api.UpdateFrontend(&scwlb.ZonedAPIUpdateFrontendRequest{ + Zone: loadbalancer.Zone, + FrontendID: frontend.ID, + Name: frontend.Name, + InboundPort: frontend.InboundPort, + BackendID: backend.ID, + TimeoutClient: frontend.TimeoutClient, + CertificateIDs: &frontend.CertificateIDs, + EnableHTTP3: frontend.EnableHTTP3, + }) + + return f, err +} + +func stringArrayEqual(got, want []string) bool { + slices.Sort(got) + slices.Sort(want) + return reflect.DeepEqual(got, want) +} + +func stringPtrArrayEqual(got, want []*string) bool { + slices.SortStableFunc(got, func(a, b *string) bool { return *a < *b }) + slices.SortStableFunc(want, func(a, b *string) bool { return *a < *b }) + return reflect.DeepEqual(got, want) +} + +func durationPtrEqual(got, want *time.Duration) bool { + if got == nil && want == nil { + return true } - return strconv.ParseBool(isPrivate) + if got == nil || want == nil { + return false + } + return *got == *want } -// Original version: https://github.com/kubernetes/legacy-cloud-providers/blob/1aa918bf227e52af6f8feb3fa065dabff251a0a3/aws/aws_loadbalancer.go#L117 -func getKeyValueFromAnnotation(annotation string) map[string]string { - additionalTags := make(map[string]string) - additionalTagsList := strings.TrimSpace(annotation) +func scwDurationPtrEqual(got, want *scw.Duration) bool { + if got == nil && want == nil { + return true + } + if got == nil || want == nil { + return false + } + return *got == *want +} - // Break up list of "Key1=Val,Key2=Val2" - tagList := strings.Split(additionalTagsList, ",") +func int32PtrEqual(got, want *int32) bool { + if got == nil && want == nil { + return true + } + if got == nil || want == nil { + return false + } + return *got == *want +} - // Break up "Key=Val" - for _, tagSet := range tagList { - tag := strings.Split(strings.TrimSpace(tagSet), "=") +func chunkArray(array []string, maxChunkSize int) [][]string { + result := [][]string{} - // Accept "Key=val" or "Key=" or just "Key" - if len(tag) >= 2 && len(tag[0]) != 0 { - // There is a key and a value, so save it - additionalTags[tag[0]] = tag[1] - } else if len(tag) == 1 && len(tag[0]) != 0 { - // Just "Key" - additionalTags[tag[0]] = "" + for len(array) > 0 { + chunkSize := maxChunkSize + if len(array) < maxChunkSize { + chunkSize = len(array) } + + result = append(result, array[:chunkSize]) + array = array[chunkSize:] } - return additionalTags + return result } -// Original version: https://github.com/kubernetes/legacy-cloud-providers/blob/1aa918bf227e52af6f8feb3fa065dabff251a0a3/aws/aws_loadbalancer.go#L1631 -func filterNodes(service *v1.Service, nodes []*v1.Node) []*v1.Node { - nodeLabels, ok := service.Annotations[serviceAnnotationLoadBalancerTargetNodeLabels] - if !ok { - return nodes +// makeACLPrefix returns the ACL prefix for rules +func makeACLPrefix(frontend *scwlb.Frontend) string { + if frontend == nil { + return "lb-source-range" } + return fmt.Sprintf("%s-lb-source-range", frontend.ID) +} - targetNodeLabels := getKeyValueFromAnnotation(nodeLabels) - - if len(targetNodeLabels) == 0 { - return nodes +func makeACLSpecs(service *v1.Service, nodes []*v1.Node, frontend *scwlb.Frontend) []*scwlb.ACLSpec { + if len(service.Spec.LoadBalancerSourceRanges) == 0 { + return []*scwlb.ACLSpec{} } - targetNodes := make([]*v1.Node, 0, len(nodes)) + aclPrefix := makeACLPrefix(frontend) + whitelist := extractNodesInternalIps(nodes) + whitelist = append(whitelist, extractNodesExternalIps(nodes)...) + whitelist = append(whitelist, service.Spec.LoadBalancerSourceRanges...) - for _, node := range nodes { - if node.Labels != nil && len(node.Labels) > 0 { - allFiltersMatch := true + slices.Sort(whitelist) - for targetLabelKey, targetLabelValue := range targetNodeLabels { - if nodeLabelValue, ok := node.Labels[targetLabelKey]; !ok || (nodeLabelValue != targetLabelValue && targetLabelValue != "") { - allFiltersMatch = false - break - } - } + subnetsChunks := chunkArray(whitelist, MaxEntriesPerACL) + acls := make([]*scwlb.ACLSpec, len(subnetsChunks)+1) - if allFiltersMatch { - targetNodes = append(targetNodes, node) - } + for idx, subnets := range subnetsChunks { + acls[idx] = &scwlb.ACLSpec{ + Name: fmt.Sprintf("%s-%d", aclPrefix, idx), + Action: &scwlb.ACLAction{ + Type: scwlb.ACLActionTypeAllow, + }, + Index: int32(idx), + Match: &scwlb.ACLMatch{ + IPSubnet: scw.StringSlicePtr(subnets), + }, } } - return targetNodes + acls[len(acls)-1] = &scwlb.ACLSpec{ + Name: fmt.Sprintf("%s-end", aclPrefix), + Action: &scwlb.ACLAction{ + Type: scwlb.ACLActionTypeDeny, + }, + Index: int32(len(acls) - 1), + Match: &scwlb.ACLMatch{ + IPSubnet: scw.StringSlicePtr([]string{"0.0.0.0/0", "::/0"}), + }, + } + + return acls } diff --git a/scaleway/loadbalancers_test.go b/scaleway/loadbalancers_test.go index 9f975f4..6cf480f 100644 --- a/scaleway/loadbalancers_test.go +++ b/scaleway/loadbalancers_test.go @@ -17,9 +17,16 @@ limitations under the License. package scaleway import ( + "encoding/json" + "reflect" + "strings" "testing" + "time" + scwlb "github.com/scaleway/scaleway-sdk-go/api/lb/v1" + "github.com/scaleway/scaleway-sdk-go/scw" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestGetValueForPort(t *testing.T) { @@ -272,3 +279,979 @@ func TestGetValueForPort(t *testing.T) { }) } } + +func TestFilterNodes(t *testing.T) { + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-target-node-labels": "key1=value1,key2=,key3,k8s.scaleway.com/pool-name=default", + }, + }, + } + nodes := []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-ok-exact-match", + Labels: map[string]string{ + "key1": "value1", + "key2": "", + "key3": "", + "k8s.scaleway.com/pool-name": "default", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-ok-with-value-in-empty-keys", + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value3", + "k8s.scaleway.com/pool-name": "default", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-ok-with-extra-keys", + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value3", + "k8s.scaleway.com/pool-name": "default", + "extra": "extra", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-ko-invalid-value-key", + Labels: map[string]string{ + "key1": "unexpected", + "key2": "value2", + "key3": "value3", + "k8s.scaleway.com/pool-name": "default", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-ko-missing-value-key", + Labels: map[string]string{ + "key2": "value2", + "key3": "value3", + "k8s.scaleway.com/pool-name": "default", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-ko-missing-empty-key", + Labels: map[string]string{ + "key1": "value1", + "key3": "value3", + "k8s.scaleway.com/pool-name": "default", + }, + }, + }, + } + + want := []string{} + for _, n := range nodes { + if strings.Contains(n.Name, "ok") { + want = append(want, n.Name) + } + } + + filteredNodes := filterNodes(service, nodes) + got := []string{} + for _, n := range filteredNodes { + got = append(got, n.Name) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("want: %s, got: %s", want, got) + } +} + +func TestServicePortToFrontend(t *testing.T) { + defaultTimeout, _ := time.ParseDuration("10m") + otherTimeout, _ := time.ParseDuration("15m") + + matrix := []struct { + name string + service *v1.Service + want *scwlb.Frontend + wantErr bool + }{ + { + "simple", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + Annotations: map[string]string{}, + }, + }, + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{}, + }, + false, + }, + { + "with timeout", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-timeout-client": "15m", + }, + }, + }, + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{}, + }, + false, + }, + { + "with one certificate", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-certificate-ids": "uid-1", + }, + }, + }, + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{"uid-1"}, + }, + false, + }, + { + "with one zoned certificate", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-certificate-ids": "fr-par-1/uid-1", + }, + }, + }, + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{"uid-1"}, + }, + false, + }, + { + "with one certificate in complex form", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-certificate-ids": "1234:uid-1;9876:uid-2", + }, + }, + }, + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{"uid-1"}, + }, + false, + }, + { + "with certificates", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-certificate-ids": "uid-1,uid-2", + }, + }, + }, + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{"uid-1", "uid-2"}, + }, + false, + }, + { + "with certificates in complex form", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-certificate-ids": "1234:uid-1,uid-2;9876:uid-3,uid-4", + }, + }, + }, + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{"uid-1", "uid-2"}, + }, + false, + }, + { + "with invalid timeout", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-timeout-client": "garbage", + }, + }, + }, + nil, + true, + }, + { + "with invalid certificate format", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-certificate-ids": "uid-1:uid-2", + }, + }, + }, + nil, + true, + }, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got, err := servicePortToFrontend(tt.service, &scwlb.LB{ID: "lbid"}, v1.ServicePort{Port: 1234}) + if tt.wantErr != (err != nil) { + t.Errorf("got error: %s, expected: %v", err, tt.wantErr) + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("want: %##v, got: %##v", tt.want, got) + } + }) + } +} + +func TestFrontendEquals(t *testing.T) { + defaultTimeout, _ := time.ParseDuration("10m") + defaultTimeout2, _ := time.ParseDuration("10m") + otherTimeout, _ := time.ParseDuration("15m") + + matrix := []struct { + name string + a *scwlb.Frontend + b *scwlb.Frontend + want bool + }{ + { + "simple", + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{}, + }, + &scwlb.Frontend{ + ID: "uid-1", + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout2, + CertificateIDs: []string{}, + }, + true, + }, + { + "with first nil", + nil, + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{}, + }, + false, + }, + { + "with last nil", + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{}, + }, + nil, + false, + }, + { + "with both nil", + nil, + nil, + true, + }, + { + "full", + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-1", "uid-2"}, + }, + &scwlb.Frontend{ + ID: "uid-1", + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-2", "uid-1"}, + }, + true, + }, + { + "with a different name", + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-1", "uid-2"}, + }, + &scwlb.Frontend{ + ID: "uid-1", + Name: "different", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-2", "uid-1"}, + }, + false, + }, + { + "with a different port", + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-1", "uid-2"}, + }, + &scwlb.Frontend{ + ID: "uid-1", + Name: "uid_tcp_1234", + InboundPort: 0, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-2", "uid-1"}, + }, + false, + }, + { + "with a different timeout", + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &defaultTimeout, + CertificateIDs: []string{"uid-1", "uid-2"}, + }, + &scwlb.Frontend{ + ID: "uid-1", + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-2", "uid-1"}, + }, + false, + }, + { + "with different certificate list", + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-1"}, + }, + &scwlb.Frontend{ + ID: "uid-1", + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-2"}, + }, + false, + }, + { + "with extra certificate in list", + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-1"}, + }, + &scwlb.Frontend{ + ID: "uid-1", + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-1", "uid-2"}, + }, + false, + }, + { + "with missing certificate in list", + &scwlb.Frontend{ + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-1", "uid-2"}, + }, + &scwlb.Frontend{ + ID: "uid-1", + Name: "uid_tcp_1234", + InboundPort: 1234, + TimeoutClient: &otherTimeout, + CertificateIDs: []string{"uid-1"}, + }, + false, + }, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got := frontendEquals(tt.a, tt.b) + if got != tt.want { + t.Errorf("want: %v, got: %v", got, tt.want) + } + }) + } +} + +func TestBackendEquals(t *testing.T) { + defaultDelay, _ := time.ParseDuration("1s") + defaultTimeout, _ := time.ParseDuration("2s") + defaultTimeoutServer, _ := time.ParseDuration("3s") + defaultTimeoutConnect, _ := time.ParseDuration("4s") + defaultTimeoutTunnel, _ := time.ParseDuration("5s") + otherDuration, _ := time.ParseDuration("50m") + boolTrue := true + var int0 int32 = 0 + var int1 int32 = 1 + var intOther int32 = 5 + + reference := &scwlb.Backend{ + Name: "name", + ForwardProtocol: "proto", + ForwardPort: 1234, + ForwardPortAlgorithm: "algo", + StickySessions: "mode", + StickySessionsCookieName: "cookie", + HealthCheck: &scwlb.HealthCheck{ + Port: 1234, + CheckDelay: &defaultDelay, + CheckTimeout: &defaultTimeout, + CheckMaxRetries: 0, + TCPConfig: &scwlb.HealthCheckTCPConfig{}, + MysqlConfig: nil, + PgsqlConfig: nil, + LdapConfig: nil, + RedisConfig: nil, + HTTPConfig: nil, + HTTPSConfig: nil, + CheckSendProxy: false, + TransientCheckDelay: &scw.Duration{Seconds: 1}, + }, + Pool: []string{"1.2.3.4", "2.3.4.5"}, + SendProxyV2: &boolTrue, + TimeoutServer: &defaultTimeoutServer, + TimeoutConnect: &defaultTimeoutConnect, + TimeoutTunnel: &defaultTimeoutTunnel, + OnMarkedDownAction: "action", + ProxyProtocol: "proxy", + RedispatchAttemptCount: &int0, + MaxRetries: &int1, + } + + matrix := []struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{ + { + "simple", + reference, + deepCloneBackend(reference), + true, + }, + { + "with first nil", + nil, + deepCloneBackend(reference), + false, + }, + { + "with last nil", + deepCloneBackend(reference), + nil, + false, + }, + { + "with both nil", + nil, + nil, + true, + }, + } + + diff := deepCloneBackend(reference) + diff.Name = "other" + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different Name", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.ForwardPort = 2345 + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different ForwardPort", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.ForwardPortAlgorithm = "other" + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different ForwardPortAlgorithm", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.ForwardProtocol = "other" + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different ForwardProtocol", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.MaxRetries = &intOther + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different MaxRetries", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.OnMarkedDownAction = "other" + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different OnMarkedDownAction", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.Pool = []string{"2.3.4.5"} + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different Pool", reference, diff, true}) + + diff = deepCloneBackend(reference) + diff.ProxyProtocol = "other" + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different ProxyProtocol", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.RedispatchAttemptCount = &intOther + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different RedispatchAttemptCount", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.StickySessions = "other" + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different StickySessions", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.StickySessionsCookieName = "other" + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different StickySessionsCookieName", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.TimeoutConnect = &otherDuration + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different TimeoutConnect", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.TimeoutServer = &otherDuration + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different TimeoutServer", reference, diff, false}) + + diff = deepCloneBackend(reference) + diff.TimeoutTunnel = &otherDuration + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with a different TimeoutTunnel", reference, diff, false}) + + httpRef := deepCloneBackend(reference) + httpRef.HealthCheck.TCPConfig = nil + httpRef.HealthCheck.HTTPConfig = &scwlb.HealthCheckHTTPConfig{ + URI: "/", + Method: "POST", + Code: scw.Int32Ptr(200), + } + httpDiff := deepCloneBackend(httpRef) + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with same HTTP healthchecks", httpRef, httpDiff, true}) + + httpDiff = deepCloneBackend(httpRef) + httpDiff.HealthCheck.HTTPConfig.Code = scw.Int32Ptr(404) + matrix = append(matrix, struct { + Name string + a *scwlb.Backend + b *scwlb.Backend + want bool + }{"with same HTTP healthchecks", httpRef, httpDiff, false}) + + for _, tt := range matrix { + t.Run(tt.Name, func(t *testing.T) { + got := backendEquals(tt.a, tt.b) + if got != tt.want { + t.Errorf("want: %v, got: %v", got, tt.want) + } + }) + } +} + +func TestStringArrayEqual(t *testing.T) { + matrix := []struct { + name string + a []string + b []string + want bool + }{ + {"same", []string{"a", "b"}, []string{"a", "b"}, true}, + {"with a different order", []string{"a", "b"}, []string{"b", "a"}, true}, + {"with a different element", []string{"a", "b"}, []string{"a", "c"}, false}, + {"with a missing element", []string{"a", "b"}, []string{"a"}, false}, + {"with an extra element", []string{"a", "b"}, []string{"a", "b", "c"}, false}, + {"with first empty", nil, []string{"a", "b"}, false}, + {"with last empty", []string{"a", "b"}, nil, false}, + {"with both empty", nil, nil, true}, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got := stringArrayEqual(tt.a, tt.b) + if got != tt.want { + t.Errorf("want: %v, got: %v", got, tt.want) + } + }) + } +} +func TestStringPtrArrayEqual(t *testing.T) { + matrix := []struct { + name string + a []*string + b []*string + want bool + }{ + {"same", scw.StringSlicePtr([]string{"a", "b"}), scw.StringSlicePtr([]string{"a", "b"}), true}, + {"with a different order", scw.StringSlicePtr([]string{"a", "b"}), scw.StringSlicePtr([]string{"b", "a"}), true}, + {"with a different element", scw.StringSlicePtr([]string{"a", "b"}), scw.StringSlicePtr([]string{"a", "c"}), false}, + {"with a missing element", scw.StringSlicePtr([]string{"a", "b"}), scw.StringSlicePtr([]string{"a"}), false}, + {"with an extra element", scw.StringSlicePtr([]string{"a", "b"}), scw.StringSlicePtr([]string{"a", "b", "c"}), false}, + {"with first empty", nil, scw.StringSlicePtr([]string{"a", "b"}), false}, + {"with last empty", scw.StringSlicePtr([]string{"a", "b"}), nil, false}, + {"with both empty", nil, nil, true}, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got := stringPtrArrayEqual(tt.a, tt.b) + if got != tt.want { + t.Errorf("want: %v, got: %v", got, tt.want) + } + }) + } +} +func TestDurationPtrEqual(t *testing.T) { + duration1, _ := time.ParseDuration("10m") + otherDuration1, _ := time.ParseDuration("10m") + duration2, _ := time.ParseDuration("5s") + + matrix := []struct { + name string + a *time.Duration + b *time.Duration + want bool + }{ + {"same", &duration1, &duration1, true}, + {"same with different ptr", &duration1, &otherDuration1, true}, + {"with first nil", &duration1, nil, false}, + {"with last nil", nil, &duration1, false}, + {"with both nil", nil, nil, true}, + {"with different values", &duration1, &duration2, false}, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got := durationPtrEqual(tt.a, tt.b) + if got != tt.want { + t.Errorf("want: %v, got: %v", got, tt.want) + } + }) + } +} +func TestScwDurationPtrEqual(t *testing.T) { + duration1 := scw.Duration{Seconds: 10, Nanos: 1000} + otherDuration1 := scw.Duration{Seconds: 10, Nanos: 1000} + duration2 := scw.Duration{Seconds: 20, Nanos: 2000} + + matrix := []struct { + name string + a *scw.Duration + b *scw.Duration + want bool + }{ + {"same", &duration1, &duration1, true}, + {"same with different ptr", &duration1, &otherDuration1, true}, + {"with first nil", &duration1, nil, false}, + {"with last nil", nil, &duration1, false}, + {"with both nil", nil, nil, true}, + {"with different values", &duration1, &duration2, false}, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got := scwDurationPtrEqual(tt.a, tt.b) + if got != tt.want { + t.Errorf("want: %v, got: %v", got, tt.want) + } + }) + } +} +func TestInt32PtrEqual(t *testing.T) { + var int1 int32 = 1 + var otherInt1 int32 = 1 + var int2 int32 = 2 + + matrix := []struct { + name string + a *int32 + b *int32 + want bool + }{ + {"same", &int1, &int1, true}, + {"same with different ptr", &int1, &otherInt1, true}, + {"with first nil", &int1, nil, false}, + {"with last nil", nil, &int1, false}, + {"with both nil", nil, nil, true}, + {"with different values", &int1, &int2, false}, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got := int32PtrEqual(tt.a, tt.b) + if got != tt.want { + t.Errorf("want: %v, got: %v", got, tt.want) + } + }) + } +} +func TestChunkArray(t *testing.T) { + matrix := []struct { + name string + array []string + want [][]string + }{ + {"nil", nil, [][]string{}}, + {"empty", []string{}, [][]string{}}, + {"less than chunksize", []string{"1", "2"}, [][]string{ + {"1", "2"}, + }}, + {"equal to chunksize", []string{"1", "2", "3"}, [][]string{ + {"1", "2", "3"}, + }}, + {"more than chunksize", []string{"1", "2", "3", "4"}, [][]string{ + {"1", "2", "3"}, + {"4"}, + }}, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got := chunkArray(tt.array, 3) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("want: %v, got: %v", got, tt.want) + } + }) + } +} +func TestMakeACLPrefix(t *testing.T) { + matrix := []struct { + name string + frontend *scwlb.Frontend + want string + }{ + {"with nil", nil, "lb-source-range"}, + {"with empty frontend", &scwlb.Frontend{}, "-lb-source-range"}, + {"with frontend", &scwlb.Frontend{ID: "uid"}, "uid-lb-source-range"}, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got := makeACLPrefix(tt.frontend) + if got != tt.want { + t.Errorf("want: %v, got: %v", got, tt.want) + } + }) + } +} + +func TestGetHTTPHealthCheck(t *testing.T) { + matrix := []struct { + name string + svc *v1.Service + want *scwlb.HealthCheckHTTPConfig + }{ + {"with empty config", &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-health-check-type": "http", + }, + }, + }, &scwlb.HealthCheckHTTPConfig{URI: "/", Method: "GET", Code: scw.Int32Ptr(200)}}, + + {"with just a domain", &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-health-check-type": "http", + "service.beta.kubernetes.io/scw-loadbalancer-health-check-http-uri": "domain.tld", + }, + }, + }, &scwlb.HealthCheckHTTPConfig{URI: "/", Method: "GET", Code: scw.Int32Ptr(200), HostHeader: "domain.tld"}}, + + {"with a domain and path", &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-health-check-type": "http", + "service.beta.kubernetes.io/scw-loadbalancer-health-check-http-uri": "domain.tld/path", + }, + }, + }, &scwlb.HealthCheckHTTPConfig{URI: "/path", Method: "GET", Code: scw.Int32Ptr(200), HostHeader: "domain.tld"}}, + + {"with a domain, path and query params", &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-health-check-type": "http", + "service.beta.kubernetes.io/scw-loadbalancer-health-check-http-uri": "domain.tld/path?password=xxxx", + }, + }, + }, &scwlb.HealthCheckHTTPConfig{URI: "/path?password=xxxx", Method: "GET", Code: scw.Int32Ptr(200), HostHeader: "domain.tld"}}, + + {"with just a path", &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-health-check-type": "http", + "service.beta.kubernetes.io/scw-loadbalancer-health-check-http-uri": "/path", + }, + }, + }, &scwlb.HealthCheckHTTPConfig{URI: "/path", Method: "GET", Code: scw.Int32Ptr(200)}}, + + {"with a specific code", &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-health-check-type": "http", + "service.beta.kubernetes.io/scw-loadbalancer-health-check-http-code": "404", + }, + }, + }, &scwlb.HealthCheckHTTPConfig{URI: "/", Method: "GET", Code: scw.Int32Ptr(404)}}, + + {"with a specific method", &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/scw-loadbalancer-health-check-type": "http", + "service.beta.kubernetes.io/scw-loadbalancer-health-check-http-method": "POST", + }, + }, + }, &scwlb.HealthCheckHTTPConfig{URI: "/", Method: "POST", Code: scw.Int32Ptr(200)}}, + } + + for _, tt := range matrix { + t.Run(tt.name, func(t *testing.T) { + got, _ := getHTTPHealthCheck(tt.svc, int32(80)) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("want: %v, got: %v", tt.want, got) + } + }) + } +} + +func deepCloneBackend(original *scwlb.Backend) *scwlb.Backend { + originalJSON, err := json.Marshal(original) + if err != nil { + panic(err) + } + + var clone *scwlb.Backend + + err = json.Unmarshal(originalJSON, &clone) + if err != nil { + panic(err) + } + + return clone +} diff --git a/scaleway/sync.go b/scaleway/sync.go index d335048..410af83 100644 --- a/scaleway/sync.go +++ b/scaleway/sync.go @@ -25,6 +25,7 @@ import ( "github.com/scaleway/scaleway-sdk-go/api/instance/v1" "github.com/scaleway/scaleway-sdk-go/api/lb/v1" "github.com/scaleway/scaleway-sdk-go/scw" + "golang.org/x/exp/maps" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/runtime" @@ -201,18 +202,19 @@ func (s *syncController) syncNodeTags(node *v1.Node) error { patcher := NewNodePatcher(s.clientSet, nodeCopied) nodeLabels := map[string]string{} - nodeTaints := []v1.Taint{} + // Note: taints must be unique by key and effect pair + nodeTaints := map[string]v1.Taint{} for _, tag := range server.Server.Tags { if strings.HasPrefix(tag, labelTaintPrefix) { key, value, effect := tagTaintParser(tag) if key == "" { continue } - nodeTaints = append(nodeTaints, v1.Taint{ + nodeTaints[fmt.Sprintf("%s:%s", key, effect)] = v1.Taint{ Key: key, Value: value, Effect: effect, - }) + } } else { var key string var value string @@ -251,12 +253,13 @@ func (s *syncController) syncNodeTags(node *v1.Node) error { } for _, taint := range node.Spec.Taints { - if !strings.HasPrefix(taint.Key, taintsPrefix) { - nodeTaints = append(nodeTaints, taint) + taintUniqueKey := fmt.Sprintf("%s:%s", taint.Key, taint.Effect) + if _, ok := nodeTaints[taintUniqueKey]; !ok && !strings.HasPrefix(taint.Key, taintsPrefix) { + nodeTaints[taintUniqueKey] = taint } } - nodeCopied.Spec.Taints = nodeTaints + nodeCopied.Spec.Taints = maps.Values(nodeTaints) err = patcher.Patch() if err != nil { klog.Errorf("error patching service: %v", err)