diff --git a/pkg/controllers/resources/pods/syncer.go b/pkg/controllers/resources/pods/syncer.go index 79ac2b884f..3c39a860bb 100644 --- a/pkg/controllers/resources/pods/syncer.go +++ b/pkg/controllers/resources/pods/syncer.go @@ -2,7 +2,9 @@ package pods import ( "context" + "fmt" "reflect" + "strings" "time" "k8s.io/apimachinery/pkg/api/equality" @@ -15,6 +17,7 @@ import ( translatepods "github.com/loft-sh/vcluster/pkg/controllers/resources/pods/translate" "github.com/loft-sh/vcluster/pkg/util/loghelper" "github.com/loft-sh/vcluster/pkg/util/toleration" + "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -85,6 +88,7 @@ func New(ctx *synccontext.RegisterContext) (syncer.Object, error) { serviceName: ctx.Options.ServiceName, enableScheduler: ctx.Options.EnableScheduler, + fakeKubeletIPs: ctx.Options.FakeKubeletIPs, virtualClusterClient: virtualClusterClient, physicalClusterClient: physicalClusterClient, @@ -102,6 +106,7 @@ type podSyncer struct { serviceName string enableScheduler bool + fakeKubeletIPs bool podTranslator translatepods.Translator virtualClusterClient kubernetes.Interface @@ -215,17 +220,32 @@ func (s *podSyncer) SyncToHost(ctx *synccontext.SyncContext, vObj client.Object) } } - // if scheduler is enabled we only sync if the pod has a node name - if s.enableScheduler && pPod.Spec.NodeName == "" { - return ctrl.Result{}, nil + if s.enableScheduler { + // if scheduler is enabled we only sync if the pod has a node name + if pPod.Spec.NodeName == "" { + return ctrl.Result{}, nil + } + + if s.fakeKubeletIPs { + nodeIP, err := s.getNodeIP(ctx, pPod.Spec.NodeName) + if err != nil { + return ctrl.Result{}, err + } + + pPod.Annotations[translatepods.HostIPAnnotation] = nodeIP + pPod.Annotations[translatepods.HostIPsAnnotation] = nodeIP + } } return s.SyncToHostCreate(ctx, vPod, pPod) } func (s *podSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj client.Object) (ctrl.Result, error) { - vPod := vObj.(*corev1.Pod) - pPod := pObj.(*corev1.Pod) + var ( + vPod = vObj.(*corev1.Pod) + pPod = pObj.(*corev1.Pod) + err error + ) // should pod get deleted? if pPod.DeletionTimestamp != nil { @@ -284,9 +304,15 @@ func (s *podSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj // has status changed? strippedPod := stripHostRewriteContainer(pPod) strippedPod = stripInjectedSidecarContainers(vPod, pPod, strippedPod) + if s.fakeKubeletIPs && strippedPod.Status.HostIP != "" { + strippedPod, err = s.rewriteFakeHostIPAddresses(ctx, strippedPod) + if err != nil { + return ctrl.Result{}, err + } + } // update readiness gates & sync status virtual -> physical - strippedPod, err := UpdateConditions(ctx, strippedPod, vPod) + strippedPod, err = UpdateConditions(ctx, strippedPod, vPod) if err != nil { return ctrl.Result{}, err } @@ -318,7 +344,7 @@ func (s *podSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj // translate services to environment variables serviceEnv := translatepods.ServicesToEnvironmentVariables(vPod.Spec.EnableServiceLinks, ptrServiceList, kubeIP) for i := range vPod.Spec.EphemeralContainers { - envVar, envFrom := translatepods.ContainerEnv(vPod.Spec.EphemeralContainers[i].Env, vPod.Spec.EphemeralContainers[i].EnvFrom, vPod, serviceEnv) + envVar, envFrom := translatepods.ContainerEnv(vPod.Spec.EphemeralContainers[i].Env, vPod.Spec.EphemeralContainers[i].EnvFrom, vPod, serviceEnv, s.fakeKubeletIPs, s.enableScheduler) vPod.Spec.EphemeralContainers[i].Env = envVar vPod.Spec.EphemeralContainers[i].EnvFrom = envFrom } @@ -493,3 +519,29 @@ func stripInjectedSidecarContainers(vPod, pPod, strippedPod *corev1.Pod) *corev1 return strippedPod } + +func (s *podSyncer) rewriteFakeHostIPAddresses(ctx *synccontext.SyncContext, strippedPod *corev1.Pod) (*corev1.Pod, error) { + nodeIP, err := s.getNodeIP(ctx, strippedPod.Spec.NodeName) + if err != nil { + return strippedPod, err + } + + strippedPod.Status.HostIP = nodeIP + strippedPod.Status.HostIPs = []corev1.HostIP{ + {IP: nodeIP}, + } + + return strippedPod, nil +} + +func (s *podSyncer) getNodeIP(ctx *synccontext.SyncContext, name string) (string, error) { + serviceName := translate.SafeConcatName(translate.VClusterName, "node", strings.ReplaceAll(name, ".", "-")) + + nodeService := &corev1.Service{} + err := ctx.CurrentNamespaceClient.Get(ctx.Context, types.NamespacedName{Name: serviceName, Namespace: ctx.CurrentNamespace}, nodeService) + if err != nil && !kerrors.IsNotFound(err) { + return "", fmt.Errorf("list services: %w", err) + } + + return nodeService.Spec.ClusterIP, nil +} diff --git a/pkg/controllers/resources/pods/syncer_test.go b/pkg/controllers/resources/pods/syncer_test.go index 65acb2f403..ddb38dfc91 100644 --- a/pkg/controllers/resources/pods/syncer_test.go +++ b/pkg/controllers/resources/pods/syncer_test.go @@ -419,6 +419,29 @@ func TestSync(t *testing.T) { maps.Copy(pPodWithLabels.Labels, convertLabelKeyWithPrefix(testLabels)) pPodWithLabels.Annotations[podtranslate.VClusterLabelsAnnotation] = podtranslate.LabelsAnnotation(vPodWithLabels) + testNodeName := "test123" + pVclusterNodeService := pVclusterService.DeepCopy() + pVclusterNodeService.Name = translate.SafeConcatName(generictesting.DefaultTestVclusterName, "node", testNodeName) + + pPodFakeKubelet := pPodBase.DeepCopy() + pPodFakeKubelet.Spec.NodeName = testNodeName + pPodFakeKubelet.Status.HostIP = "3.3.3.3" + pPodFakeKubelet.Status.HostIPs = []corev1.HostIP{ + {IP: "3.3.3.3"}, + } + + vPodWithHostIP := vPodWithNodeName.DeepCopy() + vPodWithHostIP.Status.HostIP = pVclusterService.Spec.ClusterIP + vPodWithHostIP.Status.HostIPs = []corev1.HostIP{ + {IP: pVclusterService.Spec.ClusterIP}, + } + + testNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNodeName, + }, + } + generictesting.RunTests(t, []*generictesting.SyncTest{ { Name: "Delete virtual pod", @@ -557,6 +580,21 @@ func TestSync(t *testing.T) { assert.NilError(t, err) }, }, + { + Name: "Fake Kubelet enabled with Node sync", + InitialVirtualState: []runtime.Object{testNode.DeepCopy(), vPodWithNodeName, vNamespace.DeepCopy()}, + InitialPhysicalState: []runtime.Object{testNode.DeepCopy(), pVclusterNodeService.DeepCopy(), pPodFakeKubelet.DeepCopy()}, + ExpectedVirtualState: map[schema.GroupVersionKind][]runtime.Object{ + corev1.SchemeGroupVersion.WithKind("Pod"): {vPodWithHostIP}, + }, + Sync: func(ctx *synccontext.RegisterContext) { + ctx.Options.SyncAllNodes = true + ctx.Options.FakeKubeletIPs = true + synccontext, syncer := generictesting.FakeStartSyncer(t, ctx, New) + _, err := syncer.(*podSyncer).Sync(synccontext, pPodFakeKubelet.DeepCopy(), vPodWithNodeName) + assert.NilError(t, err) + }, + }, }) } diff --git a/pkg/controllers/resources/pods/translate/translator.go b/pkg/controllers/resources/pods/translate/translator.go index f1f3bb7f01..ad5255ccea 100644 --- a/pkg/controllers/resources/pods/translate/translator.go +++ b/pkg/controllers/resources/pods/translate/translator.go @@ -43,6 +43,8 @@ const ( ClusterAutoScalerDaemonSetAnnotation = "cluster-autoscaler.kubernetes.io/daemonset-pod" ServiceAccountNameAnnotation = "vcluster.loft.sh/service-account-name" ServiceAccountTokenAnnotation = "vcluster.loft.sh/token-" + HostIPAnnotation = "vcluster.loft.sh/host-ip" + HostIPsAnnotation = "vcluster.loft.sh/host-ips" ) var ( @@ -91,6 +93,7 @@ func NewTranslator(ctx *synccontext.RegisterContext, eventRecorder record.EventR serviceAccountsEnabled: ctx.Controllers.Has("serviceaccounts"), priorityClassesEnabled: ctx.Controllers.Has("priorityclasses"), enableScheduler: ctx.Options.EnableScheduler, + fakeKubeletIPs: ctx.Options.FakeKubeletIPs, syncedLabels: ctx.Options.SyncLabels, mountPhysicalHostPaths: ctx.Options.MountPhysicalHostPaths, @@ -119,6 +122,7 @@ type translator struct { overrideHostsImage string priorityClassesEnabled bool enableScheduler bool + fakeKubeletIPs bool syncedLabels []string mountPhysicalHostPaths bool @@ -271,7 +275,7 @@ func (t *translator) Translate(ctx context.Context, vPod *corev1.Pod, services [ // translate containers for i := range pPod.Spec.Containers { - envVar, envFrom := ContainerEnv(pPod.Spec.Containers[i].Env, pPod.Spec.Containers[i].EnvFrom, vPod, serviceEnv) + envVar, envFrom := ContainerEnv(pPod.Spec.Containers[i].Env, pPod.Spec.Containers[i].EnvFrom, vPod, serviceEnv, t.fakeKubeletIPs, t.enableScheduler) pPod.Spec.Containers[i].Env = envVar pPod.Spec.Containers[i].EnvFrom = envFrom pPod.Spec.Containers[i].Image = t.imageTranslator.Translate(pPod.Spec.Containers[i].Image) @@ -279,7 +283,7 @@ func (t *translator) Translate(ctx context.Context, vPod *corev1.Pod, services [ // translate init containers for i := range pPod.Spec.InitContainers { - envVar, envFrom := ContainerEnv(pPod.Spec.InitContainers[i].Env, pPod.Spec.InitContainers[i].EnvFrom, vPod, serviceEnv) + envVar, envFrom := ContainerEnv(pPod.Spec.InitContainers[i].Env, pPod.Spec.InitContainers[i].EnvFrom, vPod, serviceEnv, t.fakeKubeletIPs, t.enableScheduler) pPod.Spec.InitContainers[i].Env = envVar pPod.Spec.InitContainers[i].EnvFrom = envFrom pPod.Spec.InitContainers[i].Image = t.imageTranslator.Translate(pPod.Spec.InitContainers[i].Image) @@ -287,7 +291,7 @@ func (t *translator) Translate(ctx context.Context, vPod *corev1.Pod, services [ // translate ephemeral containers for i := range pPod.Spec.EphemeralContainers { - envVar, envFrom := ContainerEnv(pPod.Spec.EphemeralContainers[i].Env, pPod.Spec.EphemeralContainers[i].EnvFrom, vPod, serviceEnv) + envVar, envFrom := ContainerEnv(pPod.Spec.EphemeralContainers[i].Env, pPod.Spec.EphemeralContainers[i].EnvFrom, vPod, serviceEnv, t.fakeKubeletIPs, t.enableScheduler) pPod.Spec.EphemeralContainers[i].Env = envVar pPod.Spec.EphemeralContainers[i].EnvFrom = envFrom pPod.Spec.EphemeralContainers[i].Image = t.imageTranslator.Translate(pPod.Spec.EphemeralContainers[i].Image) @@ -384,7 +388,7 @@ func (t *translator) translateVolumes(ctx context.Context, pPod *corev1.Pod, vPo } if pPod.Spec.Volumes[i].DownwardAPI != nil { for j := range pPod.Spec.Volumes[i].DownwardAPI.Items { - translateFieldRef(pPod.Spec.Volumes[i].DownwardAPI.Items[j].FieldRef) + translateFieldRef(pPod.Spec.Volumes[i].DownwardAPI.Items[j].FieldRef, t.fakeKubeletIPs, t.enableScheduler) } } if pPod.Spec.Volumes[i].ISCSI != nil && pPod.Spec.Volumes[i].ISCSI.SecretRef != nil { @@ -455,7 +459,7 @@ func (t *translator) translateProjectedVolume( } if projectedVolume.Sources[i].DownwardAPI != nil { for j := range projectedVolume.Sources[i].DownwardAPI.Items { - translateFieldRef(projectedVolume.Sources[i].DownwardAPI.Items[j].FieldRef) + translateFieldRef(projectedVolume.Sources[i].DownwardAPI.Items[j].FieldRef, t.fakeKubeletIPs, t.enableScheduler) } } if projectedVolume.Sources[i].ServiceAccountToken != nil { @@ -554,7 +558,7 @@ func (t *translator) translateProjectedVolume( return nil } -func translateFieldRef(fieldSelector *corev1.ObjectFieldSelector) { +func translateFieldRef(fieldSelector *corev1.ObjectFieldSelector, fakeKubeletIPs, enableScheduler bool) { if fieldSelector == nil { return } @@ -577,13 +581,22 @@ func translateFieldRef(fieldSelector *corev1.ObjectFieldSelector) { fieldSelector.FieldPath = "metadata.annotations['" + UIDAnnotation + "']" case "spec.serviceAccountName": fieldSelector.FieldPath = "metadata.annotations['" + ServiceAccountNameAnnotation + "']" + // translate downward API references for status.hostIP(s) only when both virtual scheduler & fakeKubeletIPs are enabled + case "status.hostIP": + if fakeKubeletIPs && enableScheduler { + fieldSelector.FieldPath = "metadata.annotations['" + HostIPAnnotation + "']" + } + case "status.hostIPs": + if fakeKubeletIPs && enableScheduler { + fieldSelector.FieldPath = "metadata.annotations['" + HostIPsAnnotation + "']" + } } } -func ContainerEnv(envVar []corev1.EnvVar, envFrom []corev1.EnvFromSource, vPod *corev1.Pod, serviceEnvMap map[string]string) ([]corev1.EnvVar, []corev1.EnvFromSource) { +func ContainerEnv(envVar []corev1.EnvVar, envFrom []corev1.EnvFromSource, vPod *corev1.Pod, serviceEnvMap map[string]string, fakeKubeletIPs, enableScheduler bool) ([]corev1.EnvVar, []corev1.EnvFromSource) { envNameMap := make(map[string]struct{}) for j, env := range envVar { - translateDownwardAPI(&envVar[j]) + translateDownwardAPI(&envVar[j], fakeKubeletIPs, enableScheduler) if env.ValueFrom != nil && env.ValueFrom.ConfigMapKeyRef != nil && env.ValueFrom.ConfigMapKeyRef.Name != "" { envVar[j].ValueFrom.ConfigMapKeyRef.Name = translate.Default.PhysicalName(envVar[j].ValueFrom.ConfigMapKeyRef.Name, vPod.Namespace) } @@ -624,14 +637,14 @@ func ContainerEnv(envVar []corev1.EnvVar, envFrom []corev1.EnvFromSource, vPod * return envVar, envFrom } -func translateDownwardAPI(env *corev1.EnvVar) { +func translateDownwardAPI(env *corev1.EnvVar, fakeKubeletIPs, enableScheduler bool) { if env.ValueFrom == nil { return } if env.ValueFrom.FieldRef == nil { return } - translateFieldRef(env.ValueFrom.FieldRef) + translateFieldRef(env.ValueFrom.FieldRef, fakeKubeletIPs, enableScheduler) } func (t *translator) translateDNSConfig(pPod *corev1.Pod, vPod *corev1.Pod, nameServer string) { diff --git a/test/e2e/syncer/pods/pods.go b/test/e2e/syncer/pods/pods.go index 2d565304a5..5f5d2b364b 100644 --- a/test/e2e/syncer/pods/pods.go +++ b/test/e2e/syncer/pods/pods.go @@ -80,6 +80,8 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() { pod, err := f.HostClient.CoreV1().Pods(translate.Default.PhysicalNamespace(ns)).Get(f.Context, translate.Default.PhysicalName(podName, ns), metav1.GetOptions{}) framework.ExpectNoError(err) + // ignore HostIP differences + resetHostIP(vpod, pod) framework.ExpectEqual(vpod.Status, pod.Status) // check for ephemeralContainers subResource @@ -137,6 +139,9 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() { framework.ExpectNoError(err) pod, err := f.HostClient.CoreV1().Pods(translate.Default.PhysicalNamespace(ns)).Get(f.Context, translate.Default.PhysicalName(podName, ns), metav1.GetOptions{}) framework.ExpectNoError(err) + + // ignore HostIP differences + resetHostIP(vpod, pod) framework.ExpectEqual(vpod.Status, pod.Status) // check for conditions @@ -542,3 +547,8 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() { } }) }) + +func resetHostIP(vpod, pod *corev1.Pod) { + vpod.Status.HostIP, pod.Status.HostIP = "", "" + vpod.Status.HostIPs, pod.Status.HostIPs = nil, nil +}