Skip to content

Commit

Permalink
[v0.19] Set fake Node IP for the virtual pod status.HostIP field when…
Browse files Browse the repository at this point in the history 
… fakeKubeletIPs is enabled (#2373)

* set fake Node IP for the virtual pod status.HostIP field when fakeKubeletIPs is enabled

* add tests

* translate the Downward API to use the correct rewritten HostIP(s) only when virtual scheduler is enabled

* move rewrite check into translateFieldRef
  • Loading branch information
@neogopher
neogopher authored Jan 22, 2025
1 parent c9216fd commit 7b31f68
Show file tree
Hide file tree
Showing 4 changed files with 130 additions and 17 deletions.
66 changes: 59 additions & 7 deletions pkg/controllers/resources/pods/syncer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package pods

import (
"context"
"fmt"
"reflect"
"strings"
"time"

"k8s.io/apimachinery/pkg/api/equality"
Expand All @@ -15,6 +17,7 @@ import (
translatepods "github.com/loft-sh/vcluster/pkg/controllers/resources/pods/translate"
"github.com/loft-sh/vcluster/pkg/util/loghelper"
"github.com/loft-sh/vcluster/pkg/util/toleration"
"github.com/loft-sh/vcluster/pkg/util/translate"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -85,6 +88,7 @@ func New(ctx *synccontext.RegisterContext) (syncer.Object, error) {

serviceName: ctx.Options.ServiceName,
enableScheduler: ctx.Options.EnableScheduler,
fakeKubeletIPs: ctx.Options.FakeKubeletIPs,

virtualClusterClient: virtualClusterClient,
physicalClusterClient: physicalClusterClient,
Expand All @@ -102,6 +106,7 @@ type podSyncer struct {

serviceName string
enableScheduler bool
fakeKubeletIPs bool

podTranslator translatepods.Translator
virtualClusterClient kubernetes.Interface
Expand Down Expand Up @@ -215,17 +220,32 @@ func (s *podSyncer) SyncToHost(ctx *synccontext.SyncContext, vObj client.Object)
}
}

// if scheduler is enabled we only sync if the pod has a node name
if s.enableScheduler && pPod.Spec.NodeName == "" {
return ctrl.Result{}, nil
if s.enableScheduler {
// if scheduler is enabled we only sync if the pod has a node name
if pPod.Spec.NodeName == "" {
return ctrl.Result{}, nil
}

if s.fakeKubeletIPs {
nodeIP, err := s.getNodeIP(ctx, pPod.Spec.NodeName)
if err != nil {
return ctrl.Result{}, err
}

pPod.Annotations[translatepods.HostIPAnnotation] = nodeIP
pPod.Annotations[translatepods.HostIPsAnnotation] = nodeIP
}
}

return s.SyncToHostCreate(ctx, vPod, pPod)
}

func (s *podSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj client.Object) (ctrl.Result, error) {
vPod := vObj.(*corev1.Pod)
pPod := pObj.(*corev1.Pod)
var (
vPod = vObj.(*corev1.Pod)
pPod = pObj.(*corev1.Pod)
err error
)

// should pod get deleted?
if pPod.DeletionTimestamp != nil {
Expand Down Expand Up @@ -284,9 +304,15 @@ func (s *podSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj
// has status changed?
strippedPod := stripHostRewriteContainer(pPod)
strippedPod = stripInjectedSidecarContainers(vPod, pPod, strippedPod)
if s.fakeKubeletIPs && strippedPod.Status.HostIP != "" {
strippedPod, err = s.rewriteFakeHostIPAddresses(ctx, strippedPod)
if err != nil {
return ctrl.Result{}, err
}
}

// update readiness gates & sync status virtual -> physical
strippedPod, err := UpdateConditions(ctx, strippedPod, vPod)
strippedPod, err = UpdateConditions(ctx, strippedPod, vPod)
if err != nil {
return ctrl.Result{}, err
}
Expand Down Expand Up @@ -318,7 +344,7 @@ func (s *podSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj
// translate services to environment variables
serviceEnv := translatepods.ServicesToEnvironmentVariables(vPod.Spec.EnableServiceLinks, ptrServiceList, kubeIP)
for i := range vPod.Spec.EphemeralContainers {
envVar, envFrom := translatepods.ContainerEnv(vPod.Spec.EphemeralContainers[i].Env, vPod.Spec.EphemeralContainers[i].EnvFrom, vPod, serviceEnv)
envVar, envFrom := translatepods.ContainerEnv(vPod.Spec.EphemeralContainers[i].Env, vPod.Spec.EphemeralContainers[i].EnvFrom, vPod, serviceEnv, s.fakeKubeletIPs, s.enableScheduler)
vPod.Spec.EphemeralContainers[i].Env = envVar
vPod.Spec.EphemeralContainers[i].EnvFrom = envFrom
}
Expand Down Expand Up @@ -493,3 +519,29 @@ func stripInjectedSidecarContainers(vPod, pPod, strippedPod *corev1.Pod) *corev1

return strippedPod
}

func (s *podSyncer) rewriteFakeHostIPAddresses(ctx *synccontext.SyncContext, strippedPod *corev1.Pod) (*corev1.Pod, error) {
nodeIP, err := s.getNodeIP(ctx, strippedPod.Spec.NodeName)
if err != nil {
return strippedPod, err
}

strippedPod.Status.HostIP = nodeIP
strippedPod.Status.HostIPs = []corev1.HostIP{
{IP: nodeIP},
}

return strippedPod, nil
}

func (s *podSyncer) getNodeIP(ctx *synccontext.SyncContext, name string) (string, error) {
serviceName := translate.SafeConcatName(translate.VClusterName, "node", strings.ReplaceAll(name, ".", "-"))

nodeService := &corev1.Service{}
err := ctx.CurrentNamespaceClient.Get(ctx.Context, types.NamespacedName{Name: serviceName, Namespace: ctx.CurrentNamespace}, nodeService)
if err != nil && !kerrors.IsNotFound(err) {
return "", fmt.Errorf("list services: %w", err)
}

return nodeService.Spec.ClusterIP, nil
}
38 changes: 38 additions & 0 deletions pkg/controllers/resources/pods/syncer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,29 @@ func TestSync(t *testing.T) {
maps.Copy(pPodWithLabels.Labels, convertLabelKeyWithPrefix(testLabels))
pPodWithLabels.Annotations[podtranslate.VClusterLabelsAnnotation] = podtranslate.LabelsAnnotation(vPodWithLabels)

testNodeName := "test123"
pVclusterNodeService := pVclusterService.DeepCopy()
pVclusterNodeService.Name = translate.SafeConcatName(generictesting.DefaultTestVclusterName, "node", testNodeName)

pPodFakeKubelet := pPodBase.DeepCopy()
pPodFakeKubelet.Spec.NodeName = testNodeName
pPodFakeKubelet.Status.HostIP = "3.3.3.3"
pPodFakeKubelet.Status.HostIPs = []corev1.HostIP{
{IP: "3.3.3.3"},
}

vPodWithHostIP := vPodWithNodeName.DeepCopy()
vPodWithHostIP.Status.HostIP = pVclusterService.Spec.ClusterIP
vPodWithHostIP.Status.HostIPs = []corev1.HostIP{
{IP: pVclusterService.Spec.ClusterIP},
}

testNode := &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: testNodeName,
},
}

generictesting.RunTests(t, []*generictesting.SyncTest{
{
Name: "Delete virtual pod",
Expand Down Expand Up @@ -557,6 +580,21 @@ func TestSync(t *testing.T) {
assert.NilError(t, err)
},
},
{
Name: "Fake Kubelet enabled with Node sync",
InitialVirtualState: []runtime.Object{testNode.DeepCopy(), vPodWithNodeName, vNamespace.DeepCopy()},
InitialPhysicalState: []runtime.Object{testNode.DeepCopy(), pVclusterNodeService.DeepCopy(), pPodFakeKubelet.DeepCopy()},
ExpectedVirtualState: map[schema.GroupVersionKind][]runtime.Object{
corev1.SchemeGroupVersion.WithKind("Pod"): {vPodWithHostIP},
},
Sync: func(ctx *synccontext.RegisterContext) {
ctx.Options.SyncAllNodes = true
ctx.Options.FakeKubeletIPs = true
synccontext, syncer := generictesting.FakeStartSyncer(t, ctx, New)
_, err := syncer.(*podSyncer).Sync(synccontext, pPodFakeKubelet.DeepCopy(), vPodWithNodeName)
assert.NilError(t, err)
},
},
})
}

Expand Down
33 changes: 23 additions & 10 deletions pkg/controllers/resources/pods/translate/translator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ const (
ClusterAutoScalerDaemonSetAnnotation = "cluster-autoscaler.kubernetes.io/daemonset-pod"
ServiceAccountNameAnnotation = "vcluster.loft.sh/service-account-name"
ServiceAccountTokenAnnotation = "vcluster.loft.sh/token-"
HostIPAnnotation = "vcluster.loft.sh/host-ip"
HostIPsAnnotation = "vcluster.loft.sh/host-ips"
)

var (
Expand Down Expand Up @@ -91,6 +93,7 @@ func NewTranslator(ctx *synccontext.RegisterContext, eventRecorder record.EventR
serviceAccountsEnabled: ctx.Controllers.Has("serviceaccounts"),
priorityClassesEnabled: ctx.Controllers.Has("priorityclasses"),
enableScheduler: ctx.Options.EnableScheduler,
fakeKubeletIPs: ctx.Options.FakeKubeletIPs,
syncedLabels: ctx.Options.SyncLabels,

mountPhysicalHostPaths: ctx.Options.MountPhysicalHostPaths,
Expand Down Expand Up @@ -119,6 +122,7 @@ type translator struct {
overrideHostsImage string
priorityClassesEnabled bool
enableScheduler bool
fakeKubeletIPs bool
syncedLabels []string

mountPhysicalHostPaths bool
Expand Down Expand Up @@ -271,23 +275,23 @@ func (t *translator) Translate(ctx context.Context, vPod *corev1.Pod, services [

// translate containers
for i := range pPod.Spec.Containers {
envVar, envFrom := ContainerEnv(pPod.Spec.Containers[i].Env, pPod.Spec.Containers[i].EnvFrom, vPod, serviceEnv)
envVar, envFrom := ContainerEnv(pPod.Spec.Containers[i].Env, pPod.Spec.Containers[i].EnvFrom, vPod, serviceEnv, t.fakeKubeletIPs, t.enableScheduler)
pPod.Spec.Containers[i].Env = envVar
pPod.Spec.Containers[i].EnvFrom = envFrom
pPod.Spec.Containers[i].Image = t.imageTranslator.Translate(pPod.Spec.Containers[i].Image)
}

// translate init containers
for i := range pPod.Spec.InitContainers {
envVar, envFrom := ContainerEnv(pPod.Spec.InitContainers[i].Env, pPod.Spec.InitContainers[i].EnvFrom, vPod, serviceEnv)
envVar, envFrom := ContainerEnv(pPod.Spec.InitContainers[i].Env, pPod.Spec.InitContainers[i].EnvFrom, vPod, serviceEnv, t.fakeKubeletIPs, t.enableScheduler)
pPod.Spec.InitContainers[i].Env = envVar
pPod.Spec.InitContainers[i].EnvFrom = envFrom
pPod.Spec.InitContainers[i].Image = t.imageTranslator.Translate(pPod.Spec.InitContainers[i].Image)
}

// translate ephemeral containers
for i := range pPod.Spec.EphemeralContainers {
envVar, envFrom := ContainerEnv(pPod.Spec.EphemeralContainers[i].Env, pPod.Spec.EphemeralContainers[i].EnvFrom, vPod, serviceEnv)
envVar, envFrom := ContainerEnv(pPod.Spec.EphemeralContainers[i].Env, pPod.Spec.EphemeralContainers[i].EnvFrom, vPod, serviceEnv, t.fakeKubeletIPs, t.enableScheduler)
pPod.Spec.EphemeralContainers[i].Env = envVar
pPod.Spec.EphemeralContainers[i].EnvFrom = envFrom
pPod.Spec.EphemeralContainers[i].Image = t.imageTranslator.Translate(pPod.Spec.EphemeralContainers[i].Image)
Expand Down Expand Up @@ -384,7 +388,7 @@ func (t *translator) translateVolumes(ctx context.Context, pPod *corev1.Pod, vPo
}
if pPod.Spec.Volumes[i].DownwardAPI != nil {
for j := range pPod.Spec.Volumes[i].DownwardAPI.Items {
translateFieldRef(pPod.Spec.Volumes[i].DownwardAPI.Items[j].FieldRef)
translateFieldRef(pPod.Spec.Volumes[i].DownwardAPI.Items[j].FieldRef, t.fakeKubeletIPs, t.enableScheduler)
}
}
if pPod.Spec.Volumes[i].ISCSI != nil && pPod.Spec.Volumes[i].ISCSI.SecretRef != nil {
Expand Down Expand Up @@ -455,7 +459,7 @@ func (t *translator) translateProjectedVolume(
}
if projectedVolume.Sources[i].DownwardAPI != nil {
for j := range projectedVolume.Sources[i].DownwardAPI.Items {
translateFieldRef(projectedVolume.Sources[i].DownwardAPI.Items[j].FieldRef)
translateFieldRef(projectedVolume.Sources[i].DownwardAPI.Items[j].FieldRef, t.fakeKubeletIPs, t.enableScheduler)
}
}
if projectedVolume.Sources[i].ServiceAccountToken != nil {
Expand Down Expand Up @@ -554,7 +558,7 @@ func (t *translator) translateProjectedVolume(
return nil
}

func translateFieldRef(fieldSelector *corev1.ObjectFieldSelector) {
func translateFieldRef(fieldSelector *corev1.ObjectFieldSelector, fakeKubeletIPs, enableScheduler bool) {
if fieldSelector == nil {
return
}
Expand All @@ -577,13 +581,22 @@ func translateFieldRef(fieldSelector *corev1.ObjectFieldSelector) {
fieldSelector.FieldPath = "metadata.annotations['" + UIDAnnotation + "']"
case "spec.serviceAccountName":
fieldSelector.FieldPath = "metadata.annotations['" + ServiceAccountNameAnnotation + "']"
// translate downward API references for status.hostIP(s) only when both virtual scheduler & fakeKubeletIPs are enabled
case "status.hostIP":
if fakeKubeletIPs && enableScheduler {
fieldSelector.FieldPath = "metadata.annotations['" + HostIPAnnotation + "']"
}
case "status.hostIPs":
if fakeKubeletIPs && enableScheduler {
fieldSelector.FieldPath = "metadata.annotations['" + HostIPsAnnotation + "']"
}
}
}

func ContainerEnv(envVar []corev1.EnvVar, envFrom []corev1.EnvFromSource, vPod *corev1.Pod, serviceEnvMap map[string]string) ([]corev1.EnvVar, []corev1.EnvFromSource) {
func ContainerEnv(envVar []corev1.EnvVar, envFrom []corev1.EnvFromSource, vPod *corev1.Pod, serviceEnvMap map[string]string, fakeKubeletIPs, enableScheduler bool) ([]corev1.EnvVar, []corev1.EnvFromSource) {
envNameMap := make(map[string]struct{})
for j, env := range envVar {
translateDownwardAPI(&envVar[j])
translateDownwardAPI(&envVar[j], fakeKubeletIPs, enableScheduler)
if env.ValueFrom != nil && env.ValueFrom.ConfigMapKeyRef != nil && env.ValueFrom.ConfigMapKeyRef.Name != "" {
envVar[j].ValueFrom.ConfigMapKeyRef.Name = translate.Default.PhysicalName(envVar[j].ValueFrom.ConfigMapKeyRef.Name, vPod.Namespace)
}
Expand Down Expand Up @@ -624,14 +637,14 @@ func ContainerEnv(envVar []corev1.EnvVar, envFrom []corev1.EnvFromSource, vPod *
return envVar, envFrom
}

func translateDownwardAPI(env *corev1.EnvVar) {
func translateDownwardAPI(env *corev1.EnvVar, fakeKubeletIPs, enableScheduler bool) {
if env.ValueFrom == nil {
return
}
if env.ValueFrom.FieldRef == nil {
return
}
translateFieldRef(env.ValueFrom.FieldRef)
translateFieldRef(env.ValueFrom.FieldRef, fakeKubeletIPs, enableScheduler)
}

func (t *translator) translateDNSConfig(pPod *corev1.Pod, vPod *corev1.Pod, nameServer string) {
Expand Down
10 changes: 10 additions & 0 deletions test/e2e/syncer/pods/pods.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() {
pod, err := f.HostClient.CoreV1().Pods(translate.Default.PhysicalNamespace(ns)).Get(f.Context, translate.Default.PhysicalName(podName, ns), metav1.GetOptions{})
framework.ExpectNoError(err)

// ignore HostIP differences
resetHostIP(vpod, pod)
framework.ExpectEqual(vpod.Status, pod.Status)

// check for ephemeralContainers subResource
Expand Down Expand Up @@ -137,6 +139,9 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() {
framework.ExpectNoError(err)
pod, err := f.HostClient.CoreV1().Pods(translate.Default.PhysicalNamespace(ns)).Get(f.Context, translate.Default.PhysicalName(podName, ns), metav1.GetOptions{})
framework.ExpectNoError(err)

// ignore HostIP differences
resetHostIP(vpod, pod)
framework.ExpectEqual(vpod.Status, pod.Status)

// check for conditions
Expand Down Expand Up @@ -542,3 +547,8 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() {
}
})
})

func resetHostIP(vpod, pod *corev1.Pod) {
vpod.Status.HostIP, pod.Status.HostIP = "", ""
vpod.Status.HostIPs, pod.Status.HostIPs = nil, nil
}

0 comments on commit 7b31f68

Please sign in to comment.