[WIP] Add support for Drop-in config dir in OCP

go.mod

@@ -347,4 +347,7 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
-replace k8s.io/kube-openapi => github.com/openshift/kube-openapi v0.0.0-20230816122517-ffc8f001abb0
+replace (
+	github.com/openshift/api => github.com/sohankunkerkar/api v0.0.0-20250122221134-1e2d17ed0eb8
+	k8s.io/kube-openapi => github.com/openshift/kube-openapi v0.0.0-20230816122517-ffc8f001abb0
+)

go.sum

@@ -594,8 +594,6 @@ github.com/opencontainers/runc v1.1.13 h1:98S2srgG9vw0zWcDpFMn5TRrh8kLxa/5OFUstu
 github.com/opencontainers/runc v1.1.13/go.mod h1:R016aXacfp/gwQBYw2FDGa9m+n6atbLWrYY8hNMT/sA=
 github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg=
 github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/openshift/api v0.0.0-20250102185430-d6d8306a24ec h1:VEDRGJmiYeN0V0xW1aI9wfzEMgaMZOVasy3FzEz27Lo=
-github.com/openshift/api v0.0.0-20250102185430-d6d8306a24ec/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
 github.com/openshift/client-go v0.0.0-20241001162912-da6d55e4611f h1:FRc0bVNWprihWS0GqQWzb3dY4dkCwpOP3mDw5NwSoR4=
 github.com/openshift/client-go v0.0.0-20241001162912-da6d55e4611f/go.mod h1:KiZi2mJRH1TOJ3FtBDYS6YvUL30s/iIXaGSUrSa36mo=
 github.com/openshift/kube-openapi v0.0.0-20230816122517-ffc8f001abb0 h1:GPlAy197Jkr+D0T2FNWanamraTdzS/r9ZkT29lxvHaA=
@@ -706,6 +704,8 @@ github.com/sivchari/tenv v1.7.1 h1:PSpuD4bu6fSmtWMxSGWcvqUUgIn7k3yOJhOIzVWn8Ak=
 github.com/sivchari/tenv v1.7.1/go.mod h1:64yStXKSOxDfX47NlhVwND4dHwfZDdbp2Lyl018Icvg=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
+github.com/sohankunkerkar/api v0.0.0-20250122221134-1e2d17ed0eb8 h1:xpBYiw78AUYWg2k8MkcgEG2TiVWXx9J/rykMmcYXDOc=
+github.com/sohankunkerkar/api v0.0.0-20250122221134-1e2d17ed0eb8/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
 github.com/sonatard/noctx v0.0.2 h1:L7Dz4De2zDQhW8S0t+KUjY0MAQJd6SgVwhzNIc4ok00=
 github.com/sonatard/noctx v0.0.2/go.mod h1:kzFz+CzWSjQ2OzIm46uJZoXuBpa2+0y3T36U18dWqIo=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=

pkg/controller/kubelet-config/helpers.go

@@ -3,7 +3,9 @@ package kubeletconfig
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 
@@ -20,6 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/klog/v2"
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
+	yamlv2 "sigs.k8s.io/yaml"
 
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
 	mcfgclientset "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
@@ -81,6 +84,12 @@ func createNewKubeletLogLevelIgnition(level int32) *ign3types.File {
 	return &r
 }
 
+func createNewKubeletDropInDirConfigIgnition(dir string) *ign3types.File {
+	config := fmt.Sprintf("[Service]\nEnvironment=\"KUBELET_CONFIG_DROPIN_DIR=%s\"\n", dir)
+	r := ctrlcommon.NewIgnFileBytesOverwriting("/etc/systemd/system/kubelet.service.d/30-dropin-dir.conf", []byte(config))
+	return &r
+}
+
 func createNewKubeletIgnition(yamlConfig []byte) *ign3types.File {
 
 	r := ctrlcommon.NewIgnFileBytesOverwriting("/etc/kubernetes/kubelet.conf", yamlConfig)
@@ -358,19 +367,37 @@ func validateUserKubeletConfig(cfg *mcfgv1.KubeletConfig) error {
 	if cfg.Spec.LogLevel != nil && (*cfg.Spec.LogLevel < 1 || *cfg.Spec.LogLevel > 10) {
 		return fmt.Errorf("KubeletConfig's LogLevel is not valid [1,10]: %v", cfg.Spec.LogLevel)
 	}
-	if cfg.Spec.KubeletConfig == nil || cfg.Spec.KubeletConfig.Raw == nil {
+
+	if (cfg.Spec.KubeletConfig == nil || cfg.Spec.KubeletConfig.Raw == nil) && cfg.Spec.DropInConfig == nil {
 		return nil
 	}
-	kcDecoded, err := DecodeKubeletConfig(cfg.Spec.KubeletConfig.Raw)
+
+	// If KubeletConfig is set, ensure DropInConfig is not set
+	if cfg.Spec.KubeletConfig != nil && cfg.Spec.KubeletConfig.Raw != nil {
+		// Validate the main KubeletConfig
+		if err := validateKubeletConfig(cfg, cfg.Spec.KubeletConfig); err != nil {
+			return err
+		}
+	}
+
+	// If DropInConfig is set, validate the KubeletConfig inside it
+	if cfg.Spec.DropInConfig != nil {
+		// Ensure KubeletConfig is set in DropInConfig
+		if err := validateKubeletConfig(cfg, &cfg.Spec.DropInConfig.KubeletConfig); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func validateKubeletConfig(cfg *mcfgv1.KubeletConfig, kubeletConfig *runtime.RawExtension) error {
+	kcDecoded, err := DecodeKubeletConfig(kubeletConfig.Raw)
 	if err != nil {
 		return fmt.Errorf("KubeletConfig could not be unmarshalled, err: %w", err)
 	}
 
 	// Check all the fields a user cannot set within the KubeletConfig CR.
-	// If a user were to set these values, the system may become unrecoverable
-	// (ie: not recover after a reboot).
-	// Therefore, if the KubeletConfig CR instance contains a non-zero or non-empty value
-	// for one of the following fields, the MCC will not apply the CR and error out instead.
 	if kcDecoded.CgroupDriver != "" {
 		return fmt.Errorf("KubeletConfiguration: cgroupDriver is not allowed to be set, but contains: %s", kcDecoded.CgroupDriver)
 	}
@@ -390,6 +417,7 @@ func validateUserKubeletConfig(cfg *mcfgv1.KubeletConfig) error {
 		cfg.Spec.AutoSizingReserved != nil && *cfg.Spec.AutoSizingReserved {
 		return fmt.Errorf("KubeletConfiguration: autoSizingReserved and systemdReserved cannot be set together")
 	}
+
 	return nil
 }
 
@@ -460,58 +488,59 @@ func kubeletConfigToIgnFile(cfg *kubeletconfigv1beta1.KubeletConfiguration) (*ig
 }
 
 // generateKubeletIgnFiles generates the Ignition files from the kubelet config
-func generateKubeletIgnFiles(kubeletConfig *mcfgv1.KubeletConfig, originalKubeConfig *kubeletconfigv1beta1.KubeletConfiguration) (*ign3types.File, *ign3types.File, *ign3types.File, error) {
+func generateKubeletIgnFiles(kubeletConfig *mcfgv1.KubeletConfig, originalKubeConfig *kubeletconfigv1beta1.KubeletConfiguration) (*ign3types.File, *ign3types.File, *ign3types.File, *ign3types.File, error) {
 	var (
 		kubeletIgnition            *ign3types.File
 		logLevelIgnition           *ign3types.File
 		autoSizingReservedIgnition *ign3types.File
+		dropInDirConfigIgnition    *ign3types.File
 	)
 	userDefinedSystemReserved := make(map[string]string)
 
 	if kubeletConfig.Spec.KubeletConfig != nil && kubeletConfig.Spec.KubeletConfig.Raw != nil {
-		specKubeletConfig, err := DecodeKubeletConfig(kubeletConfig.Spec.KubeletConfig.Raw)
+		specKubeletConfig, err := processDropInConfig(kubeletConfig, originalKubeConfig, userDefinedSystemReserved)
 		if err != nil {
-			return nil, nil, nil, fmt.Errorf("could not deserialize the new Kubelet config: %w", err)
+			return nil, nil, nil, nil, fmt.Errorf("could not process drop-in config: %w", err)
 		}
-
-		if val, ok := specKubeletConfig.SystemReserved["memory"]; ok {
-			userDefinedSystemReserved["memory"] = val
-			delete(specKubeletConfig.SystemReserved, "memory")
+		// Merge the Old and New
+		err = mergo.Merge(originalKubeConfig, specKubeletConfig, mergo.WithOverride)
+		if err != nil {
+			return nil, nil, nil, nil, fmt.Errorf("could not merge original config and new config: %w", err)
 		}
+	}
 
-		if val, ok := specKubeletConfig.SystemReserved["cpu"]; ok {
-			userDefinedSystemReserved["cpu"] = val
-			delete(specKubeletConfig.SystemReserved, "cpu")
+	if kubeletConfig.Spec.DropInConfig != nil {
+		dropInDir := kubeletConfig.Spec.DropInConfig.ConfigDirectory
+		dropInConfig := kubeletConfig.Spec.DropInConfig.KubeletConfig
+		cfg, err := processDropInConfig(kubeletConfig, originalKubeConfig, userDefinedSystemReserved)
+		if err != nil {
+			return nil, nil, nil, nil, fmt.Errorf("could not process drop-in config: %w", err)
 		}
-
-		if val, ok := specKubeletConfig.SystemReserved["ephemeral-storage"]; ok {
-			userDefinedSystemReserved["ephemeral-storage"] = val
-			delete(specKubeletConfig.SystemReserved, "ephemeral-storage")
+		// Decode JSON Raw into KubeletConfiguration
+		var kubeletConfigObj kubeletconfigv1beta1.KubeletConfiguration
+		err = json.Unmarshal(dropInConfig.Raw, &cfg)
+		if err != nil {
+			return nil, nil, nil, nil, fmt.Errorf("could not decode drop-in config JSON: %w", err)
 		}
 
-		// FeatureGates must be set from the FeatureGate.
-		// Remove them here to prevent the specKubeletConfig merge overwriting them.
-		specKubeletConfig.FeatureGates = nil
-
-		// "protectKernelDefaults" is a boolean, optional field with `omitempty` json tag in the upstream kubelet configuration
-		// This field has been set to `true` by default in OCP recently
-		// As this field is an optional one with the above tag, it gets omitted when a user inputs it to `false`
-		// Reference: https://github.com/golang/go/issues/13284
-		// Adding a workaround to decide if the user has actually set the field to `false`
-		if strings.Contains(string(kubeletConfig.Spec.KubeletConfig.Raw), protectKernelDefaultsStr) {
-			originalKubeConfig.ProtectKernelDefaults = false
-		}
-		// Merge the Old and New
-		err = mergo.Merge(originalKubeConfig, specKubeletConfig, mergo.WithOverride)
+		// Convert the KubeletConfiguration into YAML
+		kubeletConfigYaml, err := yamlv2.Marshal(kubeletConfigObj)
 		if err != nil {
-			return nil, nil, nil, fmt.Errorf("could not merge original config and new config: %w", err)
+			return nil, nil, nil, nil, fmt.Errorf("could not marshal KubeletConfiguration to YAML: %w", err)
+		}
+		// Write the specKubeletConfig to a file in the drop-in directory
+		dropInFilePath := fmt.Sprintf("%s/%s", dropInDir, kubeletConfig.Spec.DropInConfig.ConfigFile)
+
+		// Write the YAML data to the specified file
+		if err := os.WriteFile(dropInFilePath, kubeletConfigYaml, 0644); err != nil {
+			return nil, nil, nil, nil, fmt.Errorf("could not write drop-in config to file: %w", err)
 		}
 	}
 
 	// Encode the new config into an Ignition File
 	kubeletIgnition, err := kubeletConfigToIgnFile(originalKubeConfig)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("could not encode JSON: %w", err)
+		return nil, nil, nil, nil, fmt.Errorf("could not encode JSON: %w", err)
 	}
 
 	if kubeletConfig.Spec.LogLevel != nil {
@@ -523,6 +552,42 @@ func generateKubeletIgnFiles(kubeletConfig *mcfgv1.KubeletConfig, originalKubeCo
 	if len(userDefinedSystemReserved) > 0 {
 		autoSizingReservedIgnition = createNewKubeletDynamicSystemReservedIgnition(nil, userDefinedSystemReserved)
 	}
+	if kubeletConfig.Spec.DropInConfig != nil {
+		dropInDirConfigIgnition = createNewKubeletDropInDirConfigIgnition(kubeletConfig.Spec.DropInConfig.ConfigDirectory)
+	}
+
+	return kubeletIgnition, logLevelIgnition, autoSizingReservedIgnition, dropInDirConfigIgnition, nil
+}
+
+func processDropInConfig(
+	dropIn *mcfgv1.KubeletConfig,
+	originalKubeConfig *kubeletconfigv1beta1.KubeletConfiguration,
+	userDefinedSystemReserved map[string]string,
+) (cfg *kubeletconfigv1beta1.KubeletConfiguration, err error) {
+	specKubeletConfig, err := DecodeKubeletConfig(dropIn.Spec.KubeletConfig.Raw)
+	if err != nil {
+		return nil, fmt.Errorf("could not deserialize the new Kubelet config: %w", err)
+	}
+
+	for _, resource := range []string{"memory", "cpu", "ephemeral-storage"} {
+		if val, ok := specKubeletConfig.SystemReserved[resource]; ok {
+			userDefinedSystemReserved[resource] = val
+			delete(specKubeletConfig.SystemReserved, resource)
+		}
+	}
+
+	// FeatureGates must be set from the FeatureGate.
+	// Remove them here to prevent the specKubeletConfig merge overwriting them.
+	specKubeletConfig.FeatureGates = nil
+
+	// "protectKernelDefaults" is a boolean, optional field with `omitempty` json tag in the upstream kubelet configuration
+	// This field has been set to `true` by default in OCP recently
+	// As this field is an optional one with the above tag, it gets omitted when a user inputs it to `false`
+	// Reference: https://github.com/golang/go/issues/13284
+	// Adding a workaround to decide if the user has actually set the field to `false`
+	if strings.Contains(string(dropIn.Spec.KubeletConfig.Raw), protectKernelDefaultsStr) {
+		originalKubeConfig.ProtectKernelDefaults = false
+	}
 
-	return kubeletIgnition, logLevelIgnition, autoSizingReservedIgnition, nil
+	return specKubeletConfig, nil
 }

pkg/controller/kubelet-config/kubelet_config_bootstrap.go

@@ -56,7 +56,7 @@ func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletCon
 				originalKubeConfig.TLSCipherSuites = observedCipherSuites
 			}
 
-			kubeletIgnition, logLevelIgnition, autoSizingReservedIgnition, err := generateKubeletIgnFiles(kubeletConfig, originalKubeConfig)
+			kubeletIgnition, logLevelIgnition, autoSizingReservedIgnition, dropInDirConfigIgnition, err := generateKubeletIgnFiles(kubeletConfig, originalKubeConfig)
 			if err != nil {
 				return nil, err
 			}
@@ -71,6 +71,9 @@ func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletCon
 			if kubeletIgnition != nil {
 				tempIgnConfig.Storage.Files = append(tempIgnConfig.Storage.Files, *kubeletIgnition)
 			}
+			if dropInDirConfigIgnition != nil {
+				tempIgnConfig.Storage.Files = append(tempIgnConfig.Storage.Files, *dropInDirConfigIgnition)
+			}
 
 			rawIgn, err := json.Marshal(tempIgnConfig)
 			if err != nil {