From 2e39cbcfc37382f0f29f91e082920d4d0d6689a9 Mon Sep 17 00:00:00 2001 From: Sandhya Dasu Date: Mon, 27 Jan 2025 14:06:46 -0500 Subject: [PATCH 1/3] Add a template method to determine if in-cluster DNS is enabled To check if feature to run in-cluster DNS on GCP and AWS is enabled by checking if the value of `PlatformStatus.GCP.CloudLoadBalancerConfig.DNSType` is set to `ClusterHosted`. --- pkg/controller/template/render.go | 32 +++++++++++++++++++++++++++++++ pkg/operator/render.go | 28 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index f3726ea351..2fd20f39e0 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -11,6 +11,7 @@ import ( "strings" "text/template" + "github.com/sirupsen/logrus" "k8s.io/klog/v2" configv1 "github.com/openshift/api/config/v1" @@ -352,6 +353,7 @@ func renderTemplate(config RenderConfig, path string, b []byte) ([]byte, error) funcs["cloudPlatformAPIIntLoadBalancerIPs"] = cloudPlatformAPIIntLoadBalancerIPs funcs["cloudPlatformAPILoadBalancerIPs"] = cloudPlatformAPILoadBalancerIPs funcs["cloudPlatformIngressLoadBalancerIPs"] = cloudPlatformIngressLoadBalancerIPs + funcs["cloudPlatformLBIPAvailable"] = cloudPlatformLBIPAvailable funcs["join"] = strings.Join tmpl, err := template.New(path).Funcs(funcs).Parse(string(b)) if err != nil { @@ -777,6 +779,36 @@ func cloudPlatformIngressLoadBalancerIPs(cfg RenderConfig) (interface{}, error) } } +// cloudPlatformLBIPAvailable returns true when DNSType is set to `ClusterHosted` +// and LB IPs are provided as part of `PlatformStatus`. +func cloudPlatformLBIPAvailable(cfg RenderConfig) bool { + logrus.Infof("Inside cloudPlatformLBIPAvailable") + if cfg.Infra.Status.PlatformStatus != nil { + switch cfg.Infra.Status.PlatformStatus.Type { + case configv1.GCPPlatformType: + switch cloudPlatformLoadBalancerIPState(cfg) { + case availableLBIPState: + logrus.Infof("LB IPs available") + return true + default: + logrus.Infof("LB IPs not available") + return false + } + case configv1.AWSPlatformType: + switch cloudPlatformLoadBalancerIPState(cfg) { + case availableLBIPState: + return true + default: + return false + } + default: + return false + } + } else { + return false + } +} + // cloudPlatformLoadBalancerIPState is a helper function that determines if // LoadBalancer config has been set. func cloudPlatformLoadBalancerIPState(cfg RenderConfig) LoadBalancerIPState { diff --git a/pkg/operator/render.go b/pkg/operator/render.go index 73996eb4e0..14d457a106 100644 --- a/pkg/operator/render.go +++ b/pkg/operator/render.go @@ -80,6 +80,7 @@ func (a *assetRenderer) addTemplateFuncs() { funcs["cloudPlatformAPIIntLoadBalancerIPs"] = cloudPlatformAPIIntLoadBalancerIPs funcs["cloudPlatformAPILoadBalancerIPs"] = cloudPlatformAPILoadBalancerIPs funcs["cloudPlatformIngressLoadBalancerIPs"] = cloudPlatformIngressLoadBalancerIPs + funcs["cloudPlatformLBIPAvailable"] = cloudPlatformLBIPAvailable funcs["join"] = strings.Join a.tmpl = a.tmpl.Funcs(funcs) @@ -461,6 +462,33 @@ func cloudPlatformIngressLoadBalancerIPs(cfg mcfgv1.ControllerConfigSpec) (inter } } +// cloudPlatformLBIPAvailable returns true when DNSType is set to `ClusterHosted` +// and LB IPs are provided as part of `PlatformStatus`. +func cloudPlatformLBIPAvailable(cfg mcfgv1.ControllerConfigSpec) bool { + if cfg.Infra.Status.PlatformStatus != nil { + switch cfg.Infra.Status.PlatformStatus.Type { + case configv1.GCPPlatformType: + switch cloudPlatformLoadBalancerIPState(cfg) { + case availableLBIPState: + return true + default: + return false + } + case configv1.AWSPlatformType: + switch cloudPlatformLoadBalancerIPState(cfg) { + case availableLBIPState: + return true + default: + return false + } + default: + return false + } + } else { + return false + } +} + // cloudPlatformLoadBalancerIPState is a helper function that determines if // LoadBalancer config has been set. func cloudPlatformLoadBalancerIPState(cfg mcfgv1.ControllerConfigSpec) LoadBalancerIPState { From 7937ebb31aa5bef362c1eb465c8decbf3de24404 Mon Sep 17 00:00:00 2001 From: Sandhya Dasu Date: Wed, 22 Jan 2025 12:05:35 -0500 Subject: [PATCH 2/3] GCP: Update /etc/hosts file when ClusterHostedDNS is enable Append /etc/hosts files with entries to resolve cluster api and api-int URLS. /etc/hosts will provide resolution for these URLs until kubelet joins the cluster and runs its CoreDNS pod which will then take over resolution of those 2 URLs --- pkg/controller/template/render.go | 4 --- .../files/usr-local-bin-update-etc-hosts.yaml | 11 ++++++ .../units/gcp-update-etc-hosts.service.yaml | 36 +++++++++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 templates/common/gcp/files/usr-local-bin-update-etc-hosts.yaml create mode 100644 templates/common/gcp/units/gcp-update-etc-hosts.service.yaml diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index 2fd20f39e0..d57070eb88 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -11,7 +11,6 @@ import ( "strings" "text/template" - "github.com/sirupsen/logrus" "k8s.io/klog/v2" configv1 "github.com/openshift/api/config/v1" @@ -782,16 +781,13 @@ func cloudPlatformIngressLoadBalancerIPs(cfg RenderConfig) (interface{}, error) // cloudPlatformLBIPAvailable returns true when DNSType is set to `ClusterHosted` // and LB IPs are provided as part of `PlatformStatus`. func cloudPlatformLBIPAvailable(cfg RenderConfig) bool { - logrus.Infof("Inside cloudPlatformLBIPAvailable") if cfg.Infra.Status.PlatformStatus != nil { switch cfg.Infra.Status.PlatformStatus.Type { case configv1.GCPPlatformType: switch cloudPlatformLoadBalancerIPState(cfg) { case availableLBIPState: - logrus.Infof("LB IPs available") return true default: - logrus.Infof("LB IPs not available") return false } case configv1.AWSPlatformType: diff --git a/templates/common/gcp/files/usr-local-bin-update-etc-hosts.yaml b/templates/common/gcp/files/usr-local-bin-update-etc-hosts.yaml new file mode 100644 index 0000000000..b55229e62f --- /dev/null +++ b/templates/common/gcp/files/usr-local-bin-update-etc-hosts.yaml @@ -0,0 +1,11 @@ +mode: 0755 +path: "/usr/local/bin/update-etc-hosts" +contents: + inline: | + #!/bin/bash + etc_hosts_config_filename="/etc/conf.d/etc-hosts.conf" + if [ -f ${etc_hosts_config_filename} ] + then + cat /etc/conf.d/etc-hosts.conf >> /etc/hosts + echo "Done updating /etc/hosts" + fi diff --git a/templates/common/gcp/units/gcp-update-etc-hosts.service.yaml b/templates/common/gcp/units/gcp-update-etc-hosts.service.yaml new file mode 100644 index 0000000000..90c4af748e --- /dev/null +++ b/templates/common/gcp/units/gcp-update-etc-hosts.service.yaml @@ -0,0 +1,36 @@ +name: gcp-update-etc-hosts.service +enabled: {{if and (eq .Infra.Status.PlatformStatus.Type "GCP") (.Infra.Status.PlatformStatus.GCP) (.Infra.Status.PlatformStatus.GCP.CloudLoadBalancerConfig) (eq .Infra.Status.PlatformStatus.GCP.CloudLoadBalancerConfig.DNSType "ClusterHosted") }}true{{else}}false{{end}} +contents: | + [Unit] + Description=Update Default GCP /etc/hosts + # We don't need to do this on the firstboot + After=firstboot-osupdate.target + # Wait for NetworkManager to report it's online + After=NetworkManager-wait-online.service + # Run before kubelet + Before=kubelet-dependencies.target + + [Service] + # Need oneshot to delay kubelet + Type=oneshot + ExecStart=/bin/bash -c " \ + {{ if and (cloudPlatformLBIPAvailable .) (gt (len (cloudPlatformAPIIntLoadBalancerIPs .)) 0) }} \ + apiIntLBIP={{ index (cloudPlatformAPIIntLoadBalancerIPs .) 0 }} {{ end }} \ + {{ if and (cloudPlatformLBIPAvailable .) (gt (len (cloudPlatformAPILoadBalancerIPs .)) 0) }} \ + apiLBIP={{ index (cloudPlatformAPILoadBalancerIPs .) 0 }}{{ end }} \ + {{ if and (cloudPlatformLBIPAvailable .) (eq (len (cloudPlatformAPILoadBalancerIPs .)) 0) }} \ + apiLBIP={{ index (cloudPlatformAPIIntLoadBalancerIPs .) 0 }}{{ end }} \ + apiServerURL={{ .Infra.Status.APIServerURL }} \ + apiServerIntURL={{ .Infra.Status.APIServerInternalURL }} \ + apiServerHostPort=${apiServerURL#*//} \ + apiServerIntHostPort=${apiServerIntURL#*//} \ + apiServerHostname=${apiServerURL%:*}" \ + apiIntServerHostname=${apiServerIntURL%:*}" \ + mkdir -p /etc/conf.d \ + etc_hosts_config_filename="/etc/conf.d/etc-hosts.conf" \ + echo "${apiLBIP} ${apiServerHostname}" >> ${etc_hosts_config_filename} \ + echo "${apiIntLBIP} ${apiIntServerHostname}" >> ${etc_hosts_config_filename} \ + /usr/local/bin/update-etc-hosts" + + [Install] + RequiredBy=kubelet-dependencies.target From 30ea55d2ead5348a64bb24115411a820426ecfd6 Mon Sep 17 00:00:00 2001 From: Sandhya Dasu Date: Thu, 23 Jan 2025 16:24:02 -0500 Subject: [PATCH 3/3] GCP: Update test code for rendering Machine configs Added tests to accomodate GCP in-cluster DNS config --- pkg/controller/template/render_test.go | 2 ++ .../controller_config_gcp_custom_dns.yaml | 33 +++++++++++++++++++ ...ontroller_config_gcp_platform_default.yaml | 28 ++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 pkg/controller/template/test_data/controller_config_gcp_custom_dns.yaml create mode 100644 pkg/controller/template/test_data/controller_config_gcp_platform_default.yaml diff --git a/pkg/controller/template/render_test.go b/pkg/controller/template/render_test.go index db2e9d0af3..5f5225dab5 100644 --- a/pkg/controller/template/render_test.go +++ b/pkg/controller/template/render_test.go @@ -216,6 +216,8 @@ var ( "nutanix": "./test_data/controller_config_nutanix.yaml", "network-forwarding-sdn": "./test_data/controller_config_forwarding_sdn.yaml", "network-forwarding-ovn": "./test_data/controller_config_forwarding_ovn.yaml", + "gcp-custom-dns": "./test_data/controller_config_gcp_custom_dns.yaml", + "gcp-platform-default": "./test_data/controller_config_gcp_platform_default.yaml", } ) diff --git a/pkg/controller/template/test_data/controller_config_gcp_custom_dns.yaml b/pkg/controller/template/test_data/controller_config_gcp_custom_dns.yaml new file mode 100644 index 0000000000..3562e0fa23 --- /dev/null +++ b/pkg/controller/template/test_data/controller_config_gcp_custom_dns.yaml @@ -0,0 +1,33 @@ +apiVersion: "machineconfigurations.openshift.io/v1" +kind: "ControllerConfig" +spec: + clusterDNSIP: "10.3.0.10" + cloudProviderConfig: "" + etcdInitialCount: 3 + etcdCAData: ZHVtbXkgZXRjZC1jYQo= + rootCAData: ZHVtbXkgcm9vdC1jYQo= + pullSecret: + data: ZHVtbXkgZXRjZC1jYQo= + images: + etcd: image/etcd:1 + setupEtcdEnv: image/setupEtcdEnv:1 + infraImage: image/infraImage:1 + kubeClientAgentImage: image/kubeClientAgentImage:1 + infra: + apiVersion: config.openshift.io/v1 + kind: Infrastructure + status: + apiServerInternalURI: https://api-int.my-test-cluster.installer.team.coreos.systems:6443 + apiServerURL: https://api.my-test-cluster.installer.team.coreos.systems:6443 + etcdDiscoveryDomain: my-test-cluster.installer.team.coreos.systems + infrastructureName: my-test-cluster + platformStatus: + type: "GCP" + gcp: + cloudLoadBalancerConfig: + dnsType: "ClusterHosted" + clusterHosted: + apiLoadBalancerIPs: + - 20.20.20.20 + apiIntLoadBalancerIPs: + - 10.10.10.10 diff --git a/pkg/controller/template/test_data/controller_config_gcp_platform_default.yaml b/pkg/controller/template/test_data/controller_config_gcp_platform_default.yaml new file mode 100644 index 0000000000..ca299ff77a --- /dev/null +++ b/pkg/controller/template/test_data/controller_config_gcp_platform_default.yaml @@ -0,0 +1,28 @@ +apiVersion: "machineconfigurations.openshift.io/v1" +kind: "ControllerConfig" +spec: + clusterDNSIP: "10.3.0.10" + cloudProviderConfig: "" + etcdInitialCount: 3 + etcdCAData: ZHVtbXkgZXRjZC1jYQo= + rootCAData: ZHVtbXkgcm9vdC1jYQo= + pullSecret: + data: ZHVtbXkgZXRjZC1jYQo= + images: + etcd: image/etcd:1 + setupEtcdEnv: image/setupEtcdEnv:1 + infraImage: image/infraImage:1 + kubeClientAgentImage: image/kubeClientAgentImage:1 + infra: + apiVersion: config.openshift.io/v1 + kind: Infrastructure + status: + apiServerInternalURI: https://api-int.my-test-cluster.installer.team.coreos.systems:6443 + apiServerURL: https://api.my-test-cluster.installer.team.coreos.systems:6443 + etcdDiscoveryDomain: my-test-cluster.installer.team.coreos.systems + infrastructureName: my-test-cluster + platformStatus: + type: "GCP" + gcp: + cloudLoadBalancerConfig: + dnsType: "PlatformDefault"