diff --git a/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go b/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go index 24fa5856f2f6..46c4656c4007 100644 --- a/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go +++ b/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go @@ -19,7 +19,6 @@ import ( "fmt" "github.com/aws/aws-sdk-go/aws" - "github.com/pelletier/go-toml/v2" ) type Bottlerocket struct { @@ -27,14 +26,14 @@ type Bottlerocket struct { } func (b Bottlerocket) Script() (string, error) { - s, err := b.unmarshalCustomUserData() + s, err := NewBottlerocketConfig(b.CustomUserData) if err != nil { return "", fmt.Errorf("invalid UserData %w", err) } // Karpenter will overwrite settings present inside custom UserData // based on other fields specified in the provisioner s.Settings.Kubernetes.ClusterName = &b.ClusterName - s.Settings.Kubernetes.APIServer = b.ClusterEndpoint + s.Settings.Kubernetes.APIServer = &b.ClusterEndpoint s.Settings.Kubernetes.ClusterCertificate = b.CABundle s.Settings.Kubernetes.NodeLabels = b.Labels @@ -48,21 +47,9 @@ func (b Bottlerocket) Script() (string, error) { for _, taint := range b.Taints { s.Settings.Kubernetes.NodeTaints[taint.Key] = append(s.Settings.Kubernetes.NodeTaints[taint.Key], fmt.Sprintf("%s:%s", taint.Value, taint.Effect)) } - script, err := toml.Marshal(s) + script, err := s.MarshalTOML() if err != nil { return "", fmt.Errorf("constructing toml UserData %w", err) } return base64.StdEncoding.EncodeToString(script), nil } - -func (b Bottlerocket) unmarshalCustomUserData() (config, error) { - var c config - if b.CustomUserData == nil { - return c, nil - } - err := toml.Unmarshal([]byte(*b.CustomUserData), &c) - if err != nil { - return c, err - } - return c, nil -} diff --git a/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocketsettings.go b/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocketsettings.go index fefbe291bd90..a4de4e262880 100644 --- a/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocketsettings.go +++ b/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocketsettings.go @@ -14,110 +14,87 @@ limitations under the License. package bootstrap -// config is the root of the bottlerocket config, see more here https://github.com/bottlerocket-os/bottlerocket#using-user-data -type config struct { - Settings settings `toml:"settings"` +import ( + "github.com/pelletier/go-toml/v2" +) + +func NewBottlerocketConfig(userdata *string) (*BottlerocketConfig, error) { + c := &BottlerocketConfig{} + if userdata == nil { + return c, nil + } + if err := c.UnmarshalTOML([]byte(*userdata)); err != nil { + return c, err + } + return c, nil } -// This is a subset of all configuration in https://github.com/bottlerocket-os/bottlerocket/blob/develop/sources/models/src/aws-k8s-1.22/mod.rs -// These settings apply across all K8s versions that karpenter supports. -// This is currently an opinionated subset and can evolve over time -type settings struct { - Kubernetes kubernetes `toml:"kubernetes"` - HostContainers *hostContainers `toml:"host-containers,omitempty"` - AWS *awsConfig `toml:"aws,omitempty"` - Metrics *metrics `toml:"metrics,omitempty"` - Kernel *kernel `toml:"kernel,omitempty"` - ContainerRegistry *containerRegistry `toml:"container-registry,omitempty"` - Network *network `toml:"network,omitempty"` - NTP *ntp `toml:"ntp,omitempty"` -} - -// kubernetes specific configuration for bottlerocket api -type kubernetes struct { - APIServer string `toml:"api-server"` - ClusterCertificate *string `toml:"cluster-certificate"` - ClusterName *string `toml:"cluster-name"` - ClusterDNSIP *string `toml:"cluster-dns-ip,omitempty"` - NodeLabels map[string]string `toml:"node-labels,omitempty"` - NodeTaints map[string][]string `toml:"node-taints,omitempty"` - MaxPods *int `toml:"max-pods,omitempty"` - StaticPods map[string]staticPod `toml:"static-pods,omitempty"` - EvictionHard map[string]string `toml:"eviction-hard,omitempty"` - KubeReserved map[string]string `toml:"kube-reserved,omitempty"` - SystemReserved map[string]string `toml:"system-reserved,omitempty"` - AllowedUnsafeSysctls []string `toml:"allowed-unsafe-sysctls,omitempty"` - ServerTLSBootstrap *bool `toml:"server-tls-bootstrap,omitempty"` - RegistryQPS *int `toml:"registry-qps,omitempty"` - RegistryBurst *int `toml:"registry-burst,omitempty"` - EventQPS *int `toml:"event-qps,omitempty"` - EventBurst *int `toml:"event-burst,omitempty"` - KubeAPIQPS *int `toml:"kube-api-qps,omitempty"` - KubeAPIBurst *int `toml:"kube-api-burst,omitempty"` - ContainerLogMaxSize *string `toml:"container-log-max-size,omitempty"` - ContainerLogMaxFiles *int `toml:"container-log-max-files,omitempty"` - CPUManagerPolicy *string `toml:"cpu-manager-policy,omitempty"` - CPUManagerReconcilePeriod *string `toml:"cpu-manager-reconcile-period,omitempty"` - TopologyManagerScope *string `toml:"topology-manager-scope,omitempty"` - TopologyManagerPolicy *string `toml:"topology-manager-policy,omitempty"` +// BottlerocketConfig is the root of the bottlerocket config, see more here https://github.com/bottlerocket-os/bottlerocket#using-user-data +type BottlerocketConfig struct { + SettingsRaw map[string]interface{} `toml:"settings"` + Settings BottlerocketSettings `toml:"-"` } -type containerRegistry struct { - Credentials []*credential `toml:"credentials,omitempty"` +// BottlerocketSettings is a subset of all configuration in https://github.com/bottlerocket-os/bottlerocket/blob/develop/sources/models/src/aws-k8s-1.22/mod.rs +// These settings apply across all K8s versions that karpenter supports. +type BottlerocketSettings struct { + Kubernetes BottlerocketKubernetes `toml:"kubernetes"` } -type credential struct { - Registry *string `toml:"registry,omitempty"` - Auth *string `toml:"auth,omitempty"` - UserName *string `toml:"username,omitempty"` - Password *string `toml:"password,omitempty"` +// BottlerocketKubernetes is k8s specific configuration for bottlerocket api +type BottlerocketKubernetes struct { + APIServer *string `toml:"api-server"` + ClusterCertificate *string `toml:"cluster-certificate"` + ClusterName *string `toml:"cluster-name"` + ClusterDNSIP *string `toml:"cluster-dns-ip,omitempty"` + NodeLabels map[string]string `toml:"node-labels,omitempty"` + NodeTaints map[string][]string `toml:"node-taints,omitempty"` + MaxPods *int `toml:"max-pods,omitempty"` + StaticPods map[string]BottlerocketStaticPod `toml:"static-pods,omitempty"` + EvictionHard map[string]string `toml:"eviction-hard,omitempty"` + KubeReserved map[string]string `toml:"kube-reserved,omitempty"` + SystemReserved map[string]string `toml:"system-reserved,omitempty"` + AllowedUnsafeSysctls []string `toml:"allowed-unsafe-sysctls,omitempty"` + ServerTLSBootstrap *bool `toml:"server-tls-bootstrap,omitempty"` + RegistryQPS *int `toml:"registry-qps,omitempty"` + RegistryBurst *int `toml:"registry-burst,omitempty"` + EventQPS *int `toml:"event-qps,omitempty"` + EventBurst *int `toml:"event-burst,omitempty"` + KubeAPIQPS *int `toml:"kube-api-qps,omitempty"` + KubeAPIBurst *int `toml:"kube-api-burst,omitempty"` + ContainerLogMaxSize *string `toml:"container-log-max-size,omitempty"` + ContainerLogMaxFiles *int `toml:"container-log-max-files,omitempty"` + CPUManagerPolicy *string `toml:"cpu-manager-policy,omitempty"` + CPUManagerReconcilePeriod *string `toml:"cpu-manager-reconcile-period,omitempty"` + TopologyManagerScope *string `toml:"topology-manager-scope,omitempty"` + TopologyManagerPolicy *string `toml:"topology-manager-policy,omitempty"` } -type staticPod struct { +type BottlerocketStaticPod struct { Enabled *bool `toml:"enabled,omitempty"` Manifest *string `toml:"manifest,omitempty"` } -type awsConfig struct { - Region *string `toml:"region,omitempty"` -} - -type hostContainers struct { - Admin *admin `toml:"admin,omitempty"` - Control *control `toml:"control,omitempty"` -} - -type admin struct { - Enabled *bool `toml:"enabled,omitempty"` - Source *string `toml:"source,omitempty"` - Superpowered *bool `toml:"superpowered,omitempty"` - UserData *string `toml:"user-data,omitempty"` -} - -type control struct { - Enabled *bool `toml:"enabled,omitempty"` - Source *string `toml:"source,omitempty"` - Superpowered *bool `toml:"superpowered,omitempty"` -} - -type metrics struct { - MetricsURL *string `toml:"metrics-url,omitempty"` - SendMetrics *bool `toml:"send-metrics,omitempty"` - ServiceChecks []string `toml:"service-checks,omitempty"` -} - -type kernel struct { - Lockdown *string `toml:"lockdown,omitempty"` - SysCtl map[string]string `toml:"sysctl,omitempty"` -} - -type network struct { - Hostname *string `toml:"hostname,omitempty"` - HTTPSProxy *string `toml:"https-proxy,omitempty"` - NoProxy []string `toml:"no-proxy,omitempty"` - Hosts [][]interface{} `toml:"hosts,omitempty"` +func (c *BottlerocketConfig) UnmarshalTOML(data []byte) error { + // unmarshal known settings + s := struct { + Settings BottlerocketSettings `toml:"settings"` + }{} + if err := toml.Unmarshal(data, &s); err != nil { + return err + } + // unmarshal untyped settings + if err := toml.Unmarshal(data, c); err != nil { + return err + } + c.Settings = s.Settings + return nil } -type ntp struct { - TimeServers []string `toml:"time-servers,omitempty"` +func (c *BottlerocketConfig) MarshalTOML() ([]byte, error) { + if c.SettingsRaw == nil { + c.SettingsRaw = map[string]interface{}{} + } + c.SettingsRaw["kubernetes"] = c.Settings.Kubernetes + return toml.Marshal(c) } diff --git a/pkg/cloudprovider/aws/testdata/br_userdata_merged.golden b/pkg/cloudprovider/aws/testdata/br_userdata_merged.golden index 9e981da70f11..47e517225d6c 100644 --- a/pkg/cloudprovider/aws/testdata/br_userdata_merged.golden +++ b/pkg/cloudprovider/aws/testdata/br_userdata_merged.golden @@ -16,9 +16,9 @@ max-pods = 110 [settings.network] hostname = 'test.local' +hosts = [['10.0.0.0', ['test.example.com', 'test1.example.com']]] https-proxy = '1.2.3.4:8080' no-proxy = ['localhost', '127.0.0.1'] -hosts = [['10.0.0.0', ['test.example.com', 'test1.example.com']]] [settings.ntp] time-servers = ['169.254.169.123'] \ No newline at end of file