Skip to content

Commit

Permalink
feat: support all bottlerocket configuration (#2081)
Browse files Browse the repository at this point in the history 
* accept arbitrary bottlerocket settings

* just populate k8s
  • Loading branch information
@bwagner5
bwagner5 authored Jul 13, 2022
1 parent 485745b commit b1d6dd9
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 109 deletions.
19 changes: 3 additions & 16 deletions pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,21 @@ import (
"fmt"

"github.com/aws/aws-sdk-go/aws"
"github.com/pelletier/go-toml/v2"
)

type Bottlerocket struct {
Options
}

func (b Bottlerocket) Script() (string, error) {
s, err := b.unmarshalCustomUserData()
s, err := NewBottlerocketConfig(b.CustomUserData)
if err != nil {
return "", fmt.Errorf("invalid UserData %w", err)
}
// Karpenter will overwrite settings present inside custom UserData
// based on other fields specified in the provisioner
s.Settings.Kubernetes.ClusterName = &b.ClusterName
s.Settings.Kubernetes.APIServer = b.ClusterEndpoint
s.Settings.Kubernetes.APIServer = &b.ClusterEndpoint
s.Settings.Kubernetes.ClusterCertificate = b.CABundle
s.Settings.Kubernetes.NodeLabels = b.Labels

Expand All @@ -48,21 +47,9 @@ func (b Bottlerocket) Script() (string, error) {
for _, taint := range b.Taints {
s.Settings.Kubernetes.NodeTaints[taint.Key] = append(s.Settings.Kubernetes.NodeTaints[taint.Key], fmt.Sprintf("%s:%s", taint.Value, taint.Effect))
}
script, err := toml.Marshal(s)
script, err := s.MarshalTOML()
if err != nil {
return "", fmt.Errorf("constructing toml UserData %w", err)
}
return base64.StdEncoding.EncodeToString(script), nil
}

func (b Bottlerocket) unmarshalCustomUserData() (config, error) {
var c config
if b.CustomUserData == nil {
return c, nil
}
err := toml.Unmarshal([]byte(*b.CustomUserData), &c)
if err != nil {
return c, err
}
return c, nil
}
161 changes: 69 additions & 92 deletions pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocketsettings.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,110 +14,87 @@ limitations under the License.

package bootstrap

// config is the root of the bottlerocket config, see more here https://github.com/bottlerocket-os/bottlerocket#using-user-data
type config struct {
Settings settings `toml:"settings"`
import (
"github.com/pelletier/go-toml/v2"
)

func NewBottlerocketConfig(userdata *string) (*BottlerocketConfig, error) {
c := &BottlerocketConfig{}
if userdata == nil {
return c, nil
}
if err := c.UnmarshalTOML([]byte(*userdata)); err != nil {
return c, err
}
return c, nil
}

// This is a subset of all configuration in https://github.com/bottlerocket-os/bottlerocket/blob/develop/sources/models/src/aws-k8s-1.22/mod.rs
// These settings apply across all K8s versions that karpenter supports.
// This is currently an opinionated subset and can evolve over time
type settings struct {
Kubernetes kubernetes `toml:"kubernetes"`
HostContainers *hostContainers `toml:"host-containers,omitempty"`
AWS *awsConfig `toml:"aws,omitempty"`
Metrics *metrics `toml:"metrics,omitempty"`
Kernel *kernel `toml:"kernel,omitempty"`
ContainerRegistry *containerRegistry `toml:"container-registry,omitempty"`
Network *network `toml:"network,omitempty"`
NTP *ntp `toml:"ntp,omitempty"`
}

// kubernetes specific configuration for bottlerocket api
type kubernetes struct {
APIServer string `toml:"api-server"`
ClusterCertificate *string `toml:"cluster-certificate"`
ClusterName *string `toml:"cluster-name"`
ClusterDNSIP *string `toml:"cluster-dns-ip,omitempty"`
NodeLabels map[string]string `toml:"node-labels,omitempty"`
NodeTaints map[string][]string `toml:"node-taints,omitempty"`
MaxPods *int `toml:"max-pods,omitempty"`
StaticPods map[string]staticPod `toml:"static-pods,omitempty"`
EvictionHard map[string]string `toml:"eviction-hard,omitempty"`
KubeReserved map[string]string `toml:"kube-reserved,omitempty"`
SystemReserved map[string]string `toml:"system-reserved,omitempty"`
AllowedUnsafeSysctls []string `toml:"allowed-unsafe-sysctls,omitempty"`
ServerTLSBootstrap *bool `toml:"server-tls-bootstrap,omitempty"`
RegistryQPS *int `toml:"registry-qps,omitempty"`
RegistryBurst *int `toml:"registry-burst,omitempty"`
EventQPS *int `toml:"event-qps,omitempty"`
EventBurst *int `toml:"event-burst,omitempty"`
KubeAPIQPS *int `toml:"kube-api-qps,omitempty"`
KubeAPIBurst *int `toml:"kube-api-burst,omitempty"`
ContainerLogMaxSize *string `toml:"container-log-max-size,omitempty"`
ContainerLogMaxFiles *int `toml:"container-log-max-files,omitempty"`
CPUManagerPolicy *string `toml:"cpu-manager-policy,omitempty"`
CPUManagerReconcilePeriod *string `toml:"cpu-manager-reconcile-period,omitempty"`
TopologyManagerScope *string `toml:"topology-manager-scope,omitempty"`
TopologyManagerPolicy *string `toml:"topology-manager-policy,omitempty"`
// BottlerocketConfig is the root of the bottlerocket config, see more here https://github.com/bottlerocket-os/bottlerocket#using-user-data
type BottlerocketConfig struct {
SettingsRaw map[string]interface{} `toml:"settings"`
Settings BottlerocketSettings `toml:"-"`
}

type containerRegistry struct {
Credentials []*credential `toml:"credentials,omitempty"`
// BottlerocketSettings is a subset of all configuration in https://github.com/bottlerocket-os/bottlerocket/blob/develop/sources/models/src/aws-k8s-1.22/mod.rs
// These settings apply across all K8s versions that karpenter supports.
type BottlerocketSettings struct {
Kubernetes BottlerocketKubernetes `toml:"kubernetes"`
}

type credential struct {
Registry *string `toml:"registry,omitempty"`
Auth *string `toml:"auth,omitempty"`
UserName *string `toml:"username,omitempty"`
Password *string `toml:"password,omitempty"`
// BottlerocketKubernetes is k8s specific configuration for bottlerocket api
type BottlerocketKubernetes struct {
APIServer *string `toml:"api-server"`
ClusterCertificate *string `toml:"cluster-certificate"`
ClusterName *string `toml:"cluster-name"`
ClusterDNSIP *string `toml:"cluster-dns-ip,omitempty"`
NodeLabels map[string]string `toml:"node-labels,omitempty"`
NodeTaints map[string][]string `toml:"node-taints,omitempty"`
MaxPods *int `toml:"max-pods,omitempty"`
StaticPods map[string]BottlerocketStaticPod `toml:"static-pods,omitempty"`
EvictionHard map[string]string `toml:"eviction-hard,omitempty"`
KubeReserved map[string]string `toml:"kube-reserved,omitempty"`
SystemReserved map[string]string `toml:"system-reserved,omitempty"`
AllowedUnsafeSysctls []string `toml:"allowed-unsafe-sysctls,omitempty"`
ServerTLSBootstrap *bool `toml:"server-tls-bootstrap,omitempty"`
RegistryQPS *int `toml:"registry-qps,omitempty"`
RegistryBurst *int `toml:"registry-burst,omitempty"`
EventQPS *int `toml:"event-qps,omitempty"`
EventBurst *int `toml:"event-burst,omitempty"`
KubeAPIQPS *int `toml:"kube-api-qps,omitempty"`
KubeAPIBurst *int `toml:"kube-api-burst,omitempty"`
ContainerLogMaxSize *string `toml:"container-log-max-size,omitempty"`
ContainerLogMaxFiles *int `toml:"container-log-max-files,omitempty"`
CPUManagerPolicy *string `toml:"cpu-manager-policy,omitempty"`
CPUManagerReconcilePeriod *string `toml:"cpu-manager-reconcile-period,omitempty"`
TopologyManagerScope *string `toml:"topology-manager-scope,omitempty"`
TopologyManagerPolicy *string `toml:"topology-manager-policy,omitempty"`
}

type staticPod struct {
type BottlerocketStaticPod struct {
Enabled *bool `toml:"enabled,omitempty"`
Manifest *string `toml:"manifest,omitempty"`
}

type awsConfig struct {
Region *string `toml:"region,omitempty"`
}

type hostContainers struct {
Admin *admin `toml:"admin,omitempty"`
Control *control `toml:"control,omitempty"`
}

type admin struct {
Enabled *bool `toml:"enabled,omitempty"`
Source *string `toml:"source,omitempty"`
Superpowered *bool `toml:"superpowered,omitempty"`
UserData *string `toml:"user-data,omitempty"`
}

type control struct {
Enabled *bool `toml:"enabled,omitempty"`
Source *string `toml:"source,omitempty"`
Superpowered *bool `toml:"superpowered,omitempty"`
}

type metrics struct {
MetricsURL *string `toml:"metrics-url,omitempty"`
SendMetrics *bool `toml:"send-metrics,omitempty"`
ServiceChecks []string `toml:"service-checks,omitempty"`
}

type kernel struct {
Lockdown *string `toml:"lockdown,omitempty"`
SysCtl map[string]string `toml:"sysctl,omitempty"`
}

type network struct {
Hostname *string `toml:"hostname,omitempty"`
HTTPSProxy *string `toml:"https-proxy,omitempty"`
NoProxy []string `toml:"no-proxy,omitempty"`
Hosts [][]interface{} `toml:"hosts,omitempty"`
func (c *BottlerocketConfig) UnmarshalTOML(data []byte) error {
// unmarshal known settings
s := struct {
Settings BottlerocketSettings `toml:"settings"`
}{}
if err := toml.Unmarshal(data, &s); err != nil {
return err
}
// unmarshal untyped settings
if err := toml.Unmarshal(data, c); err != nil {
return err
}
c.Settings = s.Settings
return nil
}

type ntp struct {
TimeServers []string `toml:"time-servers,omitempty"`
func (c *BottlerocketConfig) MarshalTOML() ([]byte, error) {
if c.SettingsRaw == nil {
c.SettingsRaw = map[string]interface{}{}
}
c.SettingsRaw["kubernetes"] = c.Settings.Kubernetes
return toml.Marshal(c)
}
2 changes: 1 addition & 1 deletion pkg/cloudprovider/aws/testdata/br_userdata_merged.golden
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ max-pods = 110

[settings.network]
hostname = 'test.local'
hosts = [['10.0.0.0', ['test.example.com', 'test1.example.com']]]
https-proxy = '1.2.3.4:8080'
no-proxy = ['localhost', '127.0.0.1']
hosts = [['10.0.0.0', ['test.example.com', 'test1.example.com']]]

[settings.ntp]
time-servers = ['169.254.169.123']

0 comments on commit b1d6dd9

Please sign in to comment.