From 60afe4c7997670881ab8ecba71b3da03c317578e Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Thu, 2 Nov 2023 10:55:57 +0100 Subject: [PATCH 01/10] add: timezone info in image Signed-off-by: Thorsten Klein (cherry picked from commit a26441613b25d61774308744eae22fa1576a2976) Signed-off-by: Brad Davidson --- package/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package/Dockerfile b/package/Dockerfile index 7cf48853ddb6..cbecc7792a90 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -1,5 +1,5 @@ -FROM alpine:3.17 as base -RUN apk add -U ca-certificates tar zstd +FROM alpine:3.18 as base +RUN apk add -U ca-certificates tar zstd tzdata COPY build/out/data.tar.zst / RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware && \ tar -xa -C /image -f /data.tar.zst && \ @@ -8,6 +8,7 @@ RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/li FROM scratch ARG VERSION="dev" COPY --from=base /image / +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo RUN mkdir -p /etc && \ echo 'hosts: files dns' > /etc/nsswitch.conf && \ echo "PRETTY_NAME=\"K3s ${VERSION}\"" > /etc/os-release && \ From 3a82a317f5bba30a61dfa2cb2d29bc1386c9a4d1 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 14 Nov 2023 01:04:21 +0000 Subject: [PATCH 02/10] Bump kine to fix multiple issues Ref: https://github.com/k3s-io/kine/releases/tag/v0.11.0 Signed-off-by: Brad Davidson (cherry picked from commit 32a1efa408c97d70c76e29f3895a7ee8572ae385) Signed-off-by: Brad Davidson --- go.mod | 21 +++++++++++---------- go.sum | 31 ++++++++++++++++++------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 8436fd7a8f81..00998100366b 100644 --- a/go.mod +++ b/go.mod @@ -111,8 +111,8 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/json-iterator/go v1.1.12 github.com/k3s-io/helm-controller v0.15.4 - github.com/k3s-io/kine v0.10.3 - github.com/klauspost/compress v1.16.7 + github.com/k3s-io/kine v0.11.0 + github.com/klauspost/compress v1.17.2 github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 github.com/lib/pq v1.10.2 github.com/mattn/go-sqlite3 v1.14.17 @@ -143,10 +143,10 @@ require ( go.etcd.io/etcd/etcdutl/v3 v3.5.9 go.etcd.io/etcd/server/v3 v3.5.9 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.15.0 golang.org/x/net v0.17.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.13.0 + golang.org/x/sys v0.14.0 google.golang.org/grpc v1.57.0 gopkg.in/yaml.v2 v2.4.0 inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252 @@ -332,10 +332,10 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee // indirect - github.com/nats-io/jwt/v2 v2.4.1 // indirect - github.com/nats-io/nats-server/v2 v2.9.18 // indirect - github.com/nats-io/nats.go v1.27.1 // indirect - github.com/nats-io/nkeys v0.4.4 // indirect + github.com/nats-io/jwt/v2 v2.5.3 // indirect + github.com/nats-io/nats-server/v2 v2.10.5 // indirect + github.com/nats-io/nats.go v1.31.0 // indirect + github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc3 // indirect @@ -362,6 +362,7 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/tchap/go-patricia/v2 v2.3.1 // indirect + github.com/tidwall/btree v1.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect github.com/urfave/cli/v2 v2.23.5 // indirect github.com/vbatts/tar-split v0.11.5 // indirect @@ -393,8 +394,8 @@ require ( golang.org/x/mod v0.11.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.4.0 // indirect golang.org/x/tools v0.10.0 // indirect golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect diff --git a/go.sum b/go.sum index e7de76604ebb..d512f25642c6 100644 --- a/go.sum +++ b/go.sum @@ -697,8 +697,8 @@ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 h1:B3039IkTPnwQEt4tIMjC6yd6b1Q3Z9ZZ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g= github.com/k3s-io/helm-controller v0.15.4 h1:l4DWmUWpphbtwmuXGtpr5Rql/2NaCLSv4ZD8HlND9uY= github.com/k3s-io/helm-controller v0.15.4/go.mod h1:BgCPBQblj/Ect4Q7/Umf86WvyDjdG/34D+n8wfXtoeM= -github.com/k3s-io/kine v0.10.3 h1:OamjhtcQnK7zpzbiUDvXXKaAwdkXIuzr+nuyFWSC1ZA= -github.com/k3s-io/kine v0.10.3/go.mod h1:hiOK3Gj89Py+AB11YK0fxEwkdWxBvNfaMt8PRWXqh6M= +github.com/k3s-io/kine v0.11.0 h1:7tS0H9yBDxXiy1BgEEkBWLswwG/q4sARPTHdxOMz1qw= +github.com/k3s-io/kine v0.11.0/go.mod h1:tjSsWrCetgaGMTfnJW6vzqdT/qOPhF/+nUEaE+eixBA= github.com/k3s-io/klog v1.0.0-k3s2 h1:yyvD2bQbxG7m85/pvNctLX2bUDmva5kOBvuZ77tTGBA= github.com/k3s-io/klog v1.0.0-k3s2/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= github.com/k3s-io/klog/v2 v2.80.1-k3s1 h1:mGMXURxxmabQurmtRhXuQTJ9jC0pvIhESSxRSymepS8= @@ -771,8 +771,9 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= @@ -927,17 +928,17 @@ github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6Yf github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee h1:+l6i7zS8N1LOokm7dzShezI9STRGrzp0O49Pw8Jetdk= github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee/go.mod h1:EKSYvbvWAoh0hIfuZ+ieWm8u0VOTRTeDfuQvNPKRqEg= github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= -github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= -github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= +github.com/nats-io/jwt/v2 v2.5.3 h1:/9SWvzc6hTfamcgXJ3uYRpgj+QuY2aLNqRiqrKcrpEo= +github.com/nats-io/jwt/v2 v2.5.3/go.mod h1:iysuPemFcc7p4IoYots3IuELSI4EDe9Y0bQMe+I3Bf4= github.com/nats-io/nats-server/v2 v2.7.5-0.20220309212130-5c0d1999ff72/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= -github.com/nats-io/nats-server/v2 v2.9.18 h1:00muGH0qu/7NAw1b/2eFcpIvdHcTghj6PFjUVhy8zEo= -github.com/nats-io/nats-server/v2 v2.9.18/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw= +github.com/nats-io/nats-server/v2 v2.10.5 h1:hhWt6m9ja/mNnm6ixc85jCthDaiUFPaeJI79K/MD980= +github.com/nats-io/nats-server/v2 v2.10.5/go.mod h1:xUMTU4kS//SDkJCSvFwN9SyJ9nUuLhSkzB/Qz0dvjjg= github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= -github.com/nats-io/nats.go v1.27.1 h1:OuYnal9aKVSnOzLQIzf7554OXMCG7KbaTkCSBHRcSoo= -github.com/nats-io/nats.go v1.27.1/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= +github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= +github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= -github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= -github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= +github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= +github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= @@ -1181,6 +1182,8 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtse github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes= github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= +github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= +github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= @@ -1404,16 +1407,18 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= +golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From bfd811c39f4f9bc90f7cc9a7e2c8f82339e7acc7 Mon Sep 17 00:00:00 2001 From: Oliver Larsson Date: Thu, 26 Oct 2023 11:51:25 +0200 Subject: [PATCH 03/10] QoS-class resource configuration Problem: Configuring qos-class features in containerd requres a custom containerd configuration template. Solution: Look for configuration files in default locations and configure containerd to use them if they exist. Signed-off-by: Oliver Larsson (cherry picked from commit 30c8ad926d5b9965a3173ca89828730ff73312c0) Signed-off-by: Brad Davidson --- pkg/agent/config/config.go | 1 + pkg/agent/config/config_linux.go | 22 ++++++++++++++++++++++ pkg/agent/config/config_windows.go | 5 +++++ pkg/agent/templates/templates_linux.go | 6 ++++++ pkg/daemons/config/types.go | 20 +++++++++++--------- 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 4d6246beb68d..46682b84a514 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -558,6 +558,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.Containerd.Debug = envInfo.Debug applyContainerdStateAndAddress(nodeConfig) applyCRIDockerdAddress(nodeConfig) + applyContainerdQoSClassConfigFileIfPresent(envInfo, nodeConfig) nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl") nodeConfig.Certificate = servingCert diff --git a/pkg/agent/config/config_linux.go b/pkg/agent/config/config_linux.go index 70c3b654f66a..c29e0a7d8b40 100644 --- a/pkg/agent/config/config_linux.go +++ b/pkg/agent/config/config_linux.go @@ -4,9 +4,13 @@ package config import ( + "errors" + "os" "path/filepath" + "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/sirupsen/logrus" ) func applyContainerdStateAndAddress(nodeConfig *config.Node) { @@ -17,3 +21,21 @@ func applyContainerdStateAndAddress(nodeConfig *config.Node) { func applyCRIDockerdAddress(nodeConfig *config.Node) { nodeConfig.CRIDockerd.Address = "unix:///run/k3s/cri-dockerd/cri-dockerd.sock" } + +func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, nodeConfig *config.Node) { + blockioPath := filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "blockio_config.yaml") + + // Set containerd config if file exists + if _, err := os.Stat(blockioPath); !errors.Is(err, os.ErrNotExist) { + logrus.Infof("BlockIO configuration file found") + nodeConfig.Containerd.BlockIOConfig = blockioPath + } + + rdtPath := filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "rdt_config.yaml") + + // Set containerd config if file exists + if _, err := os.Stat(rdtPath); !errors.Is(err, os.ErrNotExist) { + logrus.Infof("RDT configuration file found") + nodeConfig.Containerd.RDTConfig = rdtPath + } +} diff --git a/pkg/agent/config/config_windows.go b/pkg/agent/config/config_windows.go index fb22462054f8..351d624efbaf 100644 --- a/pkg/agent/config/config_windows.go +++ b/pkg/agent/config/config_windows.go @@ -6,6 +6,7 @@ package config import ( "path/filepath" + "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/daemons/config" ) @@ -17,3 +18,7 @@ func applyContainerdStateAndAddress(nodeConfig *config.Node) { func applyCRIDockerdAddress(nodeConfig *config.Node) { nodeConfig.CRIDockerd.Address = "npipe:////.pipe/cri-dockerd" } + +func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, nodeConfig *config.Node) { + // QoS-class resource management not supported on windows. +} diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 2133fb2defcb..002f806caa1c 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -82,6 +82,12 @@ enable_keychain = true conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}" {{end}} +{{- if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }} +[plugins."io.containerd.service.v1.tasks-service"] + {{ if .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = "{{ .NodeConfig.Containerd.BlockIOConfig }}"{{end}} + {{ if .NodeConfig.Containerd.RDTConfig }}rdt_config_file = "{{ .NodeConfig.Containerd.RDTConfig }}"{{end}} +{{end}} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] runtime_type = "io.containerd.runc.v2" diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 0d6bca384d8b..60b4483ab947 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -60,15 +60,17 @@ type Node struct { } type Containerd struct { - Address string - Log string - Root string - State string - Config string - Opt string - Template string - SELinux bool - Debug bool + Address string + Log string + Root string + State string + Config string + Opt string + Template string + BlockIOConfig string + RDTConfig string + SELinux bool + Debug bool } type CRIDockerd struct { From e1a77e5ff323588c9d7230cb52761ba3f554c022 Mon Sep 17 00:00:00 2001 From: chenk008 Date: Wed, 15 Nov 2023 07:54:32 +0800 Subject: [PATCH 04/10] add agent flag disable-apiserver-lb (#8717) * add node flag disable-agent-lb * add agent flag disable-apiserver-lb Co-authored-by: Brad Davidson Signed-off-by: chenk008 (cherry picked from commit b47cbbfd42d0b339abebda70a4cee2ce91ea66ca) Signed-off-by: Brad Davidson --- pkg/cli/cmds/agent.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index f3dcafc5236b..29dce5857449 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -209,6 +209,11 @@ var ( Destination: &AgentConfig.ImageCredProvConfig, Value: "/var/lib/rancher/credentialprovider/config.yaml", } + DisableAgentLBFlag = &cli.BoolFlag{ + Name: "disable-apiserver-lb", + Usage: "(agent/networking) (experimental) Disable the agent's client-side load-balancer and connect directly to the configured server address", + Destination: &AgentConfig.DisableLoadBalancer, + } ) func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { @@ -277,6 +282,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { DockerFlag, VPNAuth, VPNAuthFile, + DisableAgentLBFlag, }, } } From 8e009c64726ce38cc4c56441e1571d70f0af17b6 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 14 Nov 2023 23:13:16 +0000 Subject: [PATCH 05/10] Fix issue with snapshot metadata configmap Omit snapshot list configmap entries for snapshots without extra metadata; reduce log level of warnings about missing s3 metadata files. Signed-off-by: Brad Davidson (cherry picked from commit 2088218c5f6cfad4e2cafd1014ad787352cfaf2b) Signed-off-by: Brad Davidson --- pkg/etcd/s3.go | 12 ++++++++++-- pkg/etcd/snapshot_controller.go | 12 ++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index abdbf8f86d1c..1e96ebabb110 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -425,10 +425,18 @@ func (s *S3) listSnapshots(ctx context.Context) (map[string]snapshotFile, error) if sf, ok := snapshots[sfKey]; ok { logrus.Debugf("Loading snapshot metadata from s3://%s/%s", s.config.EtcdS3BucketName, metadataKey) if obj, err := s.client.GetObject(ctx, s.config.EtcdS3BucketName, metadataKey, minio.GetObjectOptions{}); err != nil { - logrus.Warnf("Failed to get snapshot metadata: %v", err) + if isNotExist(err) { + logrus.Debugf("Failed to get snapshot metadata: %v", err) + } else { + logrus.Warnf("Failed to get snapshot metadata for %s: %v", filename, err) + } } else { if m, err := ioutil.ReadAll(obj); err != nil { - logrus.Warnf("Failed to read snapshot metadata: %v", err) + if isNotExist(err) { + logrus.Debugf("Failed to read snapshot metadata: %v", err) + } else { + logrus.Warnf("Failed to read snapshot metadata for %s: %v", filename, err) + } } else { sf.Metadata = base64.StdEncoding.EncodeToString(m) snapshots[sfKey] = sf diff --git a/pkg/etcd/snapshot_controller.go b/pkg/etcd/snapshot_controller.go index 6c04d3569b2c..a21da8481719 100644 --- a/pkg/etcd/snapshot_controller.go +++ b/pkg/etcd/snapshot_controller.go @@ -67,7 +67,9 @@ func (e *etcdSnapshotHandler) sync(key string, esf *apisv1.ETCDSnapshotFile) (*a } return nil, err } - if esf == nil || !esf.DeletionTimestamp.IsZero() { + + // Do not create entries for snapshots that have been deleted or do not have extra metadata + if esf == nil || !esf.DeletionTimestamp.IsZero() || len(esf.Spec.Metadata) == 0 { return nil, nil } @@ -209,10 +211,12 @@ func (e *etcdSnapshotHandler) reconcile() error { snapshots := map[string]*apisv1.ETCDSnapshotFile{} for i := range snapshotList.Items { esf := &snapshotList.Items[i] - if esf.DeletionTimestamp.IsZero() { - sfKey := generateETCDSnapshotFileConfigMapKey(*esf) - snapshots[sfKey] = esf + // Do not create entries for snapshots that have been deleted or do not have extra metadata + if !esf.DeletionTimestamp.IsZero() || len(esf.Spec.Metadata) == 0 { + continue } + sfKey := generateETCDSnapshotFileConfigMapKey(*esf) + snapshots[sfKey] = esf } // Make a copy of the configmap for change detection From c62218250ae978e219b7d415a7720308f3ebae2e Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 15 Nov 2023 00:31:15 +0000 Subject: [PATCH 06/10] Skip initial datastore reconcile during cluster-reset Signed-off-by: Brad Davidson (cherry picked from commit 7ecd5874d2eab1fb24dcfad23003f42da5be3620) Signed-off-by: Brad Davidson --- pkg/cluster/bootstrap.go | 4 ++-- pkg/daemons/control/server.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index a0f804564931..4b8edbb3bf2d 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -31,7 +31,7 @@ import ( // Bootstrap attempts to load a managed database driver, if one has been initialized or should be created/joined. // It then checks to see if the cluster needs to load bootstrap data, and if so, loads data into the // ControlRuntimeBoostrap struct, either via HTTP or from the datastore. -func (c *Cluster) Bootstrap(ctx context.Context, snapshot bool) error { +func (c *Cluster) Bootstrap(ctx context.Context, clusterReset bool) error { if err := c.assignManagedDriver(ctx); err != nil { return err } @@ -43,7 +43,7 @@ func (c *Cluster) Bootstrap(ctx context.Context, snapshot bool) error { c.shouldBootstrap = shouldBootstrap if c.managedDB != nil { - if !snapshot { + if !clusterReset { isHTTP := c.config.JoinURL != "" && c.config.Token != "" // For secondary servers, we attempt to connect and reconcile with the datastore. // If that fails we fallback to the local etcd cluster start diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index c20433f8fec8..df7a33b8d0c9 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -259,7 +259,7 @@ func prepare(ctx context.Context, config *config.Control) error { cluster := cluster.New(config) - if err := cluster.Bootstrap(ctx, false); err != nil { + if err := cluster.Bootstrap(ctx, config.ClusterReset); err != nil { return err } From 4ec87fdd3824848324c1531d51f8fa6f6c9e0809 Mon Sep 17 00:00:00 2001 From: Jason Costello Date: Wed, 15 Nov 2023 17:33:31 -0500 Subject: [PATCH 07/10] Tweaked order of ingress IPs in ServiceLB (#8711) * Tweaked order of ingress IPs in ServiceLB Previously, ingress IPs were only string-sorted when returned Sorted by IP family and string-sorted in each family as part of filterByIPFamily method * Update pkg/cloudprovider/servicelb.go * Formatting Signed-off-by: Jason Costello Co-authored-by: Brad Davidson (cherry picked from commit 07ee8549149d520610a98b9a7932f638bc508b23) Signed-off-by: Brad Davidson --- pkg/cloudprovider/servicelb.go | 5 +++-- pkg/cloudprovider/servicelb_test.go | 26 ++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pkg/cloudprovider/servicelb.go b/pkg/cloudprovider/servicelb.go index fd5b4e756528..8d1cea0739d3 100644 --- a/pkg/cloudprovider/servicelb.go +++ b/pkg/cloudprovider/servicelb.go @@ -285,8 +285,6 @@ func (k *k3s) getStatus(svc *core.Service) (*core.LoadBalancerStatus, error) { return nil, err } - sort.Strings(expectedIPs) - loadbalancer := &core.LoadBalancerStatus{} for _, ip := range expectedIPs { loadbalancer.Ingress = append(loadbalancer.Ingress, core.LoadBalancerIngress{ @@ -393,6 +391,9 @@ func filterByIPFamily(ips []string, svc *core.Service) ([]string, error) { } } + sort.Strings(ipv4Addresses) + sort.Strings(ipv6Addresses) + for _, ipFamily := range svc.Spec.IPFamilies { switch ipFamily { case core.IPv4Protocol: diff --git a/pkg/cloudprovider/servicelb_test.go b/pkg/cloudprovider/servicelb_test.go index f12baeb4938a..6a25ade126b8 100644 --- a/pkg/cloudprovider/servicelb_test.go +++ b/pkg/cloudprovider/servicelb_test.go @@ -1,6 +1,7 @@ package cloudprovider import ( + "math/rand" "reflect" "testing" @@ -8,8 +9,10 @@ import ( ) const ( - addrv4 = "1.2.3.4" - addrv6 = "2001:db8::1" + addrv4 = "1.2.3.4" + addrv4_2 = "2.3.4.5" + addrv6 = "2001:db8::1" + addrv6_2 = "3001:db8::1" ) func Test_UnitFilterByIPFamily(t *testing.T) { @@ -89,3 +92,22 @@ func Test_UnitFilterByIPFamily(t *testing.T) { }) } } + +func Test_UnitFilterByIPFamily_Ordering(t *testing.T) { + want := []string{addrv4, addrv4_2, addrv6, addrv6_2} + ips := []string{addrv4, addrv4_2, addrv6, addrv6_2} + rand.Shuffle(len(ips), func(i, j int) { + ips[i], ips[j] = ips[j], ips[i] + }) + svc := &core.Service{ + Spec: core.ServiceSpec{ + IPFamilies: []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol}, + }, + } + + got, _ := filterByIPFamily(ips, svc) + + if !reflect.DeepEqual(got, want) { + t.Errorf("filterByIPFamily() = %+v\nWant = %+v", got, want) + } +} From 169a02e4abde17dba21f80f195d8fcdcdbe3c78f Mon Sep 17 00:00:00 2001 From: Harsimran Singh Maan Date: Wed, 15 Nov 2023 14:35:31 -0800 Subject: [PATCH 08/10] Disable helm CRD installation for disable-helm-controller (#8702) * Disable helm CRD installation for disable-helm-controller The NewContext package requires config as input which would require all third-party callers to update when the new go module is published. This change only affects the behaviour of installation of helm CRDs. Existing helm crds installed in a cluster would not be removed when disable-helm-controller flag is set on the server. Addresses #8701 * address review comments * remove redundant check Signed-off-by: Harsimran Singh Maan (cherry picked from commit abc2efdd57c91184afda7c520692995a8dc36fe2) Signed-off-by: Brad Davidson --- pkg/cli/etcdsnapshot/etcd_snapshot.go | 2 +- pkg/server/context.go | 23 +++++++++++++++++------ pkg/server/server.go | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/cli/etcdsnapshot/etcd_snapshot.go b/pkg/cli/etcdsnapshot/etcd_snapshot.go index 2dc4a3e7eb4a..1f83b98c5a22 100644 --- a/pkg/cli/etcdsnapshot/etcd_snapshot.go +++ b/pkg/cli/etcdsnapshot/etcd_snapshot.go @@ -86,7 +86,7 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, config *server.Config) (*e return nil, fmt.Errorf("etcd database not found in %s", config.ControlConfig.DataDir) } - sc, err := server.NewContext(ctx, config.ControlConfig.Runtime.KubeConfigAdmin, false) + sc, err := server.NewContext(ctx, config, false) if err != nil { return nil, err } diff --git a/pkg/server/context.go b/pkg/server/context.go index 49666e31be20..93bee0c6eaae 100644 --- a/pkg/server/context.go +++ b/pkg/server/context.go @@ -38,7 +38,11 @@ func (c *Context) Start(ctx context.Context) error { return start.All(ctx, 5, c.K3s, c.Helm, c.Apps, c.Auth, c.Batch, c.Core) } -func NewContext(ctx context.Context, cfg string, forServer bool) (*Context, error) { +func NewContext(ctx context.Context, config *Config, forServer bool) (*Context, error) { + cfg := config.ControlConfig.Runtime.KubeConfigAdmin + if forServer { + cfg = config.ControlConfig.Runtime.KubeConfigSupervisor + } restConfig, err := clientcmd.BuildConfigFromFlags("", cfg) if err != nil { return nil, err @@ -53,7 +57,7 @@ func NewContext(ctx context.Context, cfg string, forServer bool) (*Context, erro var recorder record.EventRecorder if forServer { recorder = util.BuildControllerEventRecorder(k8s, version.Program+"-supervisor", metav1.NamespaceAll) - if err := crds(ctx, restConfig); err != nil { + if err := registerCrds(ctx, config, restConfig); err != nil { return nil, errors.Wrap(err, "failed to register CRDs") } } @@ -70,14 +74,21 @@ func NewContext(ctx context.Context, cfg string, forServer bool) (*Context, erro }, nil } -func crds(ctx context.Context, config *rest.Config) error { - factory, err := crd.NewFactoryFromClient(config) +func registerCrds(ctx context.Context, config *Config, restConfig *rest.Config) error { + factory, err := crd.NewFactoryFromClient(restConfig) if err != nil { return err } - types := append(helmcrd.List(), addoncrd.List()...) - factory.BatchCreateCRDs(ctx, types...) + factory.BatchCreateCRDs(ctx, crds(config)...) return factory.BatchWait() } + +func crds(config *Config) []crd.CRD { + defaultCrds := addoncrd.List() + if !config.ControlConfig.DisableHelmController { + defaultCrds = append(defaultCrds, helmcrd.List()...) + } + return defaultCrds +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 90e8084b78f3..c2d64c05bb98 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -98,7 +98,7 @@ func startOnAPIServerReady(ctx context.Context, config *Config) { func runControllers(ctx context.Context, config *Config) error { controlConfig := &config.ControlConfig - sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigSupervisor, true) + sc, err := NewContext(ctx, config, true) if err != nil { return errors.Wrap(err, "failed to create new server context") } From 0461f51670e8bc915814445d9ad6215f637b31e2 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 15 Nov 2023 01:24:58 +0000 Subject: [PATCH 09/10] Add jitter to client config retry Also: * Replaces labeled for/continue RETRY loops with wait helpers for improved readability * Pulls secrets and nodes from cache for node password verification * Migrate nodepassword tests to wrangler mocks for better code reuse Signed-off-by: Brad Davidson --- go.mod | 6 +- go.sum | 8 +- pkg/agent/config/config.go | 67 ++++++------ pkg/node/controller.go | 24 ++--- pkg/nodepassword/nodepassword.go | 18 ++-- pkg/nodepassword/nodepassword_test.go | 149 +++++++++++++------------- pkg/server/router.go | 8 +- 7 files changed, 138 insertions(+), 142 deletions(-) diff --git a/go.mod b/go.mod index 00998100366b..d4103e5ffdfc 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ replace ( github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.8 github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.10.1 - github.com/rancher/wrangler => github.com/rancher/wrangler v1.1.1-0.20230425173236-39a4707f0689 + github.com/rancher/wrangler => github.com/rancher/wrangler v1.1.1-0.20230807182002-35cb42e6a915 go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1 go.etcd.io/etcd/client/pkg/v3 => github.com/k3s-io/etcd/client/pkg/v3 v3.5.9-k3s1 go.etcd.io/etcd/client/v2 => github.com/k3s-io/etcd/client/v2 v2.305.9-k3s1 @@ -105,6 +105,7 @@ require ( github.com/go-bindata/go-bindata v3.1.2+incompatible github.com/go-sql-driver/mysql v1.7.1 github.com/go-test/deep v1.0.7 + github.com/golang/mock v1.6.0 github.com/google/cadvisor v0.47.1 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 @@ -125,7 +126,7 @@ require ( github.com/otiai10/copy v1.7.0 github.com/pkg/errors v0.9.1 github.com/rancher/dynamiclistener v0.3.6-rc2 - github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc + github.com/rancher/lasso v0.0.0-20230629200414-8a54b32e6792 github.com/rancher/remotedialer v0.3.0 github.com/rancher/wharfie v0.5.3 github.com/rancher/wrangler v1.1.1 @@ -262,7 +263,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/cel-go v0.12.7 // indirect diff --git a/go.sum b/go.sum index d512f25642c6..d106806f8c49 100644 --- a/go.sum +++ b/go.sum @@ -1070,14 +1070,14 @@ github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPH github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rancher/dynamiclistener v0.3.6-rc2 h1:Y1nai+Xv+4qqlB3c+hmrY2uBo1EcCDU9kmN5hbnmZhA= github.com/rancher/dynamiclistener v0.3.6-rc2/go.mod h1:wOh62hdJIgyqTdD/VAHO77UPKAbUsJJ5gYRjzgBL3Wo= -github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc h1:29VHrInLV4qSevvcvhBj5UhQWkPShxrxv4AahYg2Scw= -github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc/go.mod h1:dEfC9eFQigj95lv/JQ8K5e7+qQCacWs1aIA6nLxKzT8= +github.com/rancher/lasso v0.0.0-20230629200414-8a54b32e6792 h1:IaPhDqppVYX2v/nCR8j2i0nqOLD5yggzzy39QUlcqDw= +github.com/rancher/lasso v0.0.0-20230629200414-8a54b32e6792/go.mod h1:dNcwXjcqgdOuKFIVETNAPURRh3e5PAi/nWUjj+MLVZA= github.com/rancher/remotedialer v0.3.0 h1:y1EO8JCsgZo0RcqTUp6U8FXcBAv27R+TLnWRcpvX1sM= github.com/rancher/remotedialer v0.3.0/go.mod h1:BwwztuvViX2JrLLUwDlsYt5DiyUwHLlzynRwkZLAY0Q= github.com/rancher/wharfie v0.5.3 h1:6hiO26H7YTgChbLAE6JppxFRjaH3tbKfMItv/LqV0Q0= github.com/rancher/wharfie v0.5.3/go.mod h1:Ebpai7digxegLroBseeC54XRBt5we3DgFS6kAE2ho+o= -github.com/rancher/wrangler v1.1.1-0.20230425173236-39a4707f0689 h1:otb4OjgXH2b8a4C9g76jCDuTF3opjaYffZ55SiVe7KU= -github.com/rancher/wrangler v1.1.1-0.20230425173236-39a4707f0689/go.mod h1:D6Tu6oVX8aGtCHsMCtYaysgVK3ad920MTSeAu7rzb5U= +github.com/rancher/wrangler v1.1.1-0.20230807182002-35cb42e6a915 h1:CNwXnngrb5PgPe4nr7G5jS298YkhS6udMnNvyBOboFQ= +github.com/rancher/wrangler v1.1.1-0.20230807182002-35cb42e6a915/go.mod h1:0oPjv01nvzeavcmeuT0xMlGKs9IJaNk5NCPPQq1n8Ro= github.com/rasky/go-xdr v0.0.0-20170217172119-4930550ba2e2/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 46682b84a514..c3cdf316b803 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -33,6 +33,7 @@ import ( "github.com/rancher/wrangler/pkg/slice" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/wait" utilsnet "k8s.io/utils/net" ) @@ -42,22 +43,28 @@ const ( // Get returns a pointer to a completed Node configuration struct, // containing a merging of the local CLI configuration with settings from the server. +// Node configuration includes client certificates, which requires node password verification, +// so this is somewhat computationally expensive on the server side, and is retried with jitter +// to avoid having clients hammer on the server at fixed periods. // A call to this will bock until agent configuration is successfully returned by the // server. func Get(ctx context.Context, agent cmds.Agent, proxy proxy.Proxy) *config.Node { - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() -RETRY: - for { - agentConfig, err := get(ctx, &agent, proxy) + var agentConfig *config.Node + var err error + + // This would be more clear as wait.PollImmediateUntilWithContext, but that function + // does not support jittering, so we instead use wait.JitterUntilWithContext, and cancel + // the context on success. + ctx, cancel := context.WithCancel(ctx) + wait.JitterUntilWithContext(ctx, func(ctx context.Context) { + agentConfig, err = get(ctx, &agent, proxy) if err != nil { logrus.Infof("Waiting to retrieve agent configuration; server is not ready: %v", err) - for range ticker.C { - continue RETRY - } + } else { + cancel() } - return agentConfig - } + }, 5*time.Second, 1.0, true) + return agentConfig } // KubeProxyDisabled returns a bool indicating whether or not kube-proxy has been disabled in the @@ -65,42 +72,40 @@ RETRY: // after all startup hooks have completed, so a call to this will block until after the server's // readyz endpoint returns OK. func KubeProxyDisabled(ctx context.Context, node *config.Node, proxy proxy.Proxy) bool { - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() -RETRY: - for { - disabled, err := getKubeProxyDisabled(ctx, node, proxy) + var disabled bool + var err error + + wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) { + disabled, err = getKubeProxyDisabled(ctx, node, proxy) if err != nil { logrus.Infof("Waiting to retrieve kube-proxy configuration; server is not ready: %v", err) - for range ticker.C { - continue RETRY - } + return false, nil } - return disabled - } + return true, nil + }) + return disabled } // APIServers returns a list of apiserver endpoints, suitable for seeding client loadbalancer configurations. // This function will block until it can return a populated list of apiservers, or if the remote server returns // an error (indicating that it does not support this functionality). func APIServers(ctx context.Context, node *config.Node, proxy proxy.Proxy) []string { - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() -RETRY: - for { - addresses, err := getAPIServers(ctx, node, proxy) + var addresses []string + var err error + + wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) { + addresses, err = getAPIServers(ctx, node, proxy) if err != nil { logrus.Infof("Failed to retrieve list of apiservers from server: %v", err) - return nil + return false, err } if len(addresses) == 0 { logrus.Infof("Waiting for apiserver addresses") - for range ticker.C { - continue RETRY - } + return false, nil } - return addresses - } + return true, nil + }) + return addresses } type HTTPRequester func(u string, client *http.Client, username, password, token string) ([]byte, error) diff --git a/pkg/node/controller.go b/pkg/node/controller.go index 478de8ea18cb..ef111541c900 100644 --- a/pkg/node/controller.go +++ b/pkg/node/controller.go @@ -13,15 +13,14 @@ import ( func Register(ctx context.Context, modCoreDNS bool, - secretClient coreclient.SecretClient, - configMap coreclient.ConfigMapController, + secrets coreclient.SecretController, + configMaps coreclient.ConfigMapController, nodes coreclient.NodeController, ) error { h := &handler{ - modCoreDNS: modCoreDNS, - secretClient: secretClient, - configCache: configMap.Cache(), - configClient: configMap, + modCoreDNS: modCoreDNS, + secrets: secrets, + configMaps: configMaps, } nodes.OnChange(ctx, "node", h.onChange) nodes.OnRemove(ctx, "node", h.onRemove) @@ -30,10 +29,9 @@ func Register(ctx context.Context, } type handler struct { - modCoreDNS bool - secretClient coreclient.SecretClient - configCache coreclient.ConfigMapCache - configClient coreclient.ConfigMapClient + modCoreDNS bool + secrets coreclient.SecretController + configMaps coreclient.ConfigMapController } func (h *handler) onChange(key string, node *core.Node) (*core.Node, error) { @@ -78,7 +76,7 @@ func (h *handler) updateCoreDNSConfigMap(nodeName, nodeAddress string, removed b return nil } - configMapCache, err := h.configCache.Get("kube-system", "coredns") + configMapCache, err := h.configMaps.Cache().Get("kube-system", "coredns") if err != nil || configMapCache == nil { logrus.Warn(errors.Wrap(err, "Unable to fetch coredns config map")) return nil @@ -120,7 +118,7 @@ func (h *handler) updateCoreDNSConfigMap(nodeName, nodeAddress string, removed b } configMap.Data["NodeHosts"] = newHosts - if _, err := h.configClient.Update(configMap); err != nil { + if _, err := h.configMaps.Update(configMap); err != nil { return err } @@ -135,5 +133,5 @@ func (h *handler) updateCoreDNSConfigMap(nodeName, nodeAddress string, removed b } func (h *handler) removeNodePassword(nodeName string) error { - return nodepassword.Delete(h.secretClient, nodeName) + return nodepassword.Delete(h.secrets, nodeName) } diff --git a/pkg/nodepassword/nodepassword.go b/pkg/nodepassword/nodepassword.go index e1a460c08a72..defb5e848156 100644 --- a/pkg/nodepassword/nodepassword.go +++ b/pkg/nodepassword/nodepassword.go @@ -51,9 +51,9 @@ func getSecretName(nodeName string) string { return strings.ToLower(nodeName + ".node-password." + version.Program) } -func verifyHash(secretClient coreclient.SecretClient, nodeName, pass string) error { +func verifyHash(secretClient coreclient.SecretController, nodeName, pass string) error { name := getSecretName(nodeName) - secret, err := secretClient.Get(metav1.NamespaceSystem, name, metav1.GetOptions{}) + secret, err := secretClient.Cache().Get(metav1.NamespaceSystem, name) if err != nil { return &passwordError{node: nodeName, err: err} } @@ -67,7 +67,7 @@ func verifyHash(secretClient coreclient.SecretClient, nodeName, pass string) err } // Ensure will verify a node-password secret if it exists, otherwise it will create one -func Ensure(secretClient coreclient.SecretClient, nodeName, pass string) error { +func Ensure(secretClient coreclient.SecretController, nodeName, pass string) error { err := verifyHash(secretClient, nodeName, pass) if apierrors.IsNotFound(err) { var hash string @@ -88,12 +88,12 @@ func Ensure(secretClient coreclient.SecretClient, nodeName, pass string) error { } // Delete will remove a node-password secret -func Delete(secretClient coreclient.SecretClient, nodeName string) error { +func Delete(secretClient coreclient.SecretController, nodeName string) error { return secretClient.Delete(metav1.NamespaceSystem, getSecretName(nodeName), &metav1.DeleteOptions{}) } // MigrateFile moves password file entries to secrets -func MigrateFile(secretClient coreclient.SecretClient, nodeClient coreclient.NodeClient, passwordFile string) error { +func MigrateFile(secretClient coreclient.SecretController, nodeClient coreclient.NodeController, passwordFile string) error { _, err := os.Stat(passwordFile) if os.IsNotExist(err) { return nil @@ -108,11 +108,9 @@ func MigrateFile(secretClient coreclient.SecretClient, nodeClient coreclient.Nod } nodeNames := []string{} - nodeList, _ := nodeClient.List(metav1.ListOptions{}) - if nodeList != nil { - for _, node := range nodeList.Items { - nodeNames = append(nodeNames, node.Name) - } + nodeList, _ := nodeClient.Cache().List(nil) + for _, node := range nodeList { + nodeNames = append(nodeNames, node.Name) } if len(nodeNames) == 0 { nodeNames = append(nodeNames, passwd.Users()...) diff --git a/pkg/nodepassword/nodepassword_test.go b/pkg/nodepassword/nodepassword_test.go index ef8624ca91ba..d44046df06db 100644 --- a/pkg/nodepassword/nodepassword_test.go +++ b/pkg/nodepassword/nodepassword_test.go @@ -8,12 +8,13 @@ import ( "runtime" "testing" + "github.com/golang/mock/gomock" + "github.com/rancher/wrangler/pkg/generic/fake" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/watch" ) const migrateNumNodes = 10 @@ -27,20 +28,29 @@ func Test_UnitAsserts(t *testing.T) { func Test_UnitEnsureDelete(t *testing.T) { logMemUsage(t) - secretClient := &mockSecretClient{} + ctrl := gomock.NewController(t) + secretClient := fake.NewMockControllerInterface[*v1.Secret, *v1.SecretList](ctrl) + secretCache := fake.NewMockCacheInterface[*v1.Secret](ctrl) + secretStore := &mockSecretStore{} + + // Set up expected call counts for tests + // Expect to see 2 creates, any number of cache gets, and 2 deletes. + secretClient.EXPECT().Create(gomock.Any()).Times(2).DoAndReturn(secretStore.Create) + secretClient.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Times(2).DoAndReturn(secretStore.Delete) + secretClient.EXPECT().Cache().AnyTimes().Return(secretCache) + secretCache.EXPECT().Get(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(secretStore.Get) + + // Run tests assertEqual(t, Ensure(secretClient, "node1", "Hello World"), nil) assertEqual(t, Ensure(secretClient, "node1", "Hello World"), nil) assertNotEqual(t, Ensure(secretClient, "node1", "Goodbye World"), nil) - assertEqual(t, secretClient.created, 1) assertEqual(t, Delete(secretClient, "node1"), nil) assertNotEqual(t, Delete(secretClient, "node1"), nil) - assertEqual(t, secretClient.deleted, 1) assertEqual(t, Ensure(secretClient, "node1", "Hello Universe"), nil) assertNotEqual(t, Ensure(secretClient, "node1", "Hello World"), nil) assertEqual(t, Ensure(secretClient, "node1", "Hello Universe"), nil) - assertEqual(t, secretClient.created, 2) logMemUsage(t) } @@ -49,16 +59,32 @@ func Test_UnitMigrateFile(t *testing.T) { nodePasswordFile := generateNodePasswordFile(migrateNumNodes) defer os.Remove(nodePasswordFile) - secretClient := &mockSecretClient{} - nodeClient := &mockNodeClient{} + ctrl := gomock.NewController(t) + + secretClient := fake.NewMockControllerInterface[*v1.Secret, *v1.SecretList](ctrl) + secretCache := fake.NewMockCacheInterface[*v1.Secret](ctrl) + secretStore := &mockSecretStore{} + + nodeClient := fake.NewMockNonNamespacedControllerInterface[*v1.Node, *v1.NodeList](ctrl) + nodeCache := fake.NewMockNonNamespacedCacheInterface[*v1.Node](ctrl) + nodeStore := &mockNodeStore{} + // Set up expected call counts for tests + // Expect to see 1 node list, any number of cache gets, and however many + // creates as we are migrating. + secretClient.EXPECT().Create(gomock.Any()).Times(migrateNumNodes).DoAndReturn(secretStore.Create) + secretClient.EXPECT().Cache().AnyTimes().Return(secretCache) + secretCache.EXPECT().Get(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(secretStore.Get) + nodeClient.EXPECT().Cache().AnyTimes().Return(nodeCache) + nodeCache.EXPECT().List(gomock.Any()).Times(1).DoAndReturn(nodeStore.List) + + // Run tests logMemUsage(t) if err := MigrateFile(secretClient, nodeClient, nodePasswordFile); err != nil { - log.Fatal(err) + t.Fatal(err) } logMemUsage(t) - assertEqual(t, secretClient.created, migrateNumNodes) assertNotEqual(t, Ensure(secretClient, "node1", "Hello World"), nil) assertEqual(t, Ensure(secretClient, "node1", "node1"), nil) } @@ -67,25 +93,43 @@ func Test_UnitMigrateFileNodes(t *testing.T) { nodePasswordFile := generateNodePasswordFile(migrateNumNodes) defer os.Remove(nodePasswordFile) - secretClient := &mockSecretClient{} - nodeClient := &mockNodeClient{} - nodeClient.nodes = make([]v1.Node, createNumNodes, createNumNodes) - for i := range nodeClient.nodes { - nodeClient.nodes[i].Name = fmt.Sprintf("node%d", i+1) + ctrl := gomock.NewController(t) + + secretClient := fake.NewMockControllerInterface[*v1.Secret, *v1.SecretList](ctrl) + secretCache := fake.NewMockCacheInterface[*v1.Secret](ctrl) + secretStore := &mockSecretStore{} + + nodeClient := fake.NewMockNonNamespacedControllerInterface[*v1.Node, *v1.NodeList](ctrl) + nodeCache := fake.NewMockNonNamespacedCacheInterface[*v1.Node](ctrl) + nodeStore := &mockNodeStore{} + + nodeStore.nodes = make([]v1.Node, createNumNodes, createNumNodes) + for i := range nodeStore.nodes { + nodeStore.nodes[i].Name = fmt.Sprintf("node%d", i+1) } + // Set up expected call counts for tests + // Expect to see 1 node list, any number of cache gets, and however many + // creates as we are migrating - plus an extra new node at the end. + secretClient.EXPECT().Create(gomock.Any()).Times(migrateNumNodes + 1).DoAndReturn(secretStore.Create) + secretClient.EXPECT().Cache().AnyTimes().Return(secretCache) + secretCache.EXPECT().Get(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(secretStore.Get) + nodeClient.EXPECT().Cache().AnyTimes().Return(nodeCache) + nodeCache.EXPECT().List(gomock.Any()).Times(1).DoAndReturn(nodeStore.List) + + // Run tests logMemUsage(t) if err := MigrateFile(secretClient, nodeClient, nodePasswordFile); err != nil { - log.Fatal(err) + t.Fatal(err) } logMemUsage(t) - assertEqual(t, secretClient.created, createNumNodes) - for _, node := range nodeClient.nodes { + for _, node := range nodeStore.nodes { assertNotEqual(t, Ensure(secretClient, node.Name, "wrong-password"), nil) assertEqual(t, Ensure(secretClient, node.Name, node.Name), nil) } - newNode := fmt.Sprintf("node%d", createNumNodes+1) + + newNode := fmt.Sprintf("node%d", migrateNumNodes+1) assertEqual(t, Ensure(secretClient, newNode, "new-password"), nil) assertNotEqual(t, Ensure(secretClient, newNode, "wrong-password"), nil) } @@ -98,16 +142,13 @@ func Test_PasswordError(t *testing.T) { } // -------------------------- +// mock secret store interface -// mock secret client interface - -type mockSecretClient struct { +type mockSecretStore struct { entries map[string]map[string]v1.Secret - created int - deleted int } -func (m *mockSecretClient) Create(secret *v1.Secret) (*v1.Secret, error) { +func (m *mockSecretStore) Create(secret *v1.Secret) (*v1.Secret, error) { if m.entries == nil { m.entries = map[string]map[string]v1.Secret{} } @@ -117,16 +158,11 @@ func (m *mockSecretClient) Create(secret *v1.Secret) (*v1.Secret, error) { if _, ok := m.entries[secret.Namespace][secret.Name]; ok { return nil, errorAlreadyExists() } - m.created++ m.entries[secret.Namespace][secret.Name] = *secret return secret, nil } -func (m *mockSecretClient) Update(secret *v1.Secret) (*v1.Secret, error) { - return nil, errorNotImplemented() -} - -func (m *mockSecretClient) Delete(namespace, name string, options *metav1.DeleteOptions) error { +func (m *mockSecretStore) Delete(namespace, name string, options *metav1.DeleteOptions) error { if m.entries == nil { return errorNotFound() } @@ -136,12 +172,11 @@ func (m *mockSecretClient) Delete(namespace, name string, options *metav1.Delete if _, ok := m.entries[namespace][name]; !ok { return errorNotFound() } - m.deleted++ delete(m.entries[namespace], name) return nil } -func (m *mockSecretClient) Get(namespace, name string, options metav1.GetOptions) (*v1.Secret, error) { +func (m *mockSecretStore) Get(namespace, name string) (*v1.Secret, error) { if m.entries == nil { return nil, errorNotFound() } @@ -154,53 +189,18 @@ func (m *mockSecretClient) Get(namespace, name string, options metav1.GetOptions return nil, errorNotFound() } -func (m *mockSecretClient) List(namespace string, opts metav1.ListOptions) (*v1.SecretList, error) { - return nil, errorNotImplemented() -} - -func (m *mockSecretClient) Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error) { - return nil, errorNotImplemented() -} - -func (m *mockSecretClient) Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Secret, err error) { - return nil, errorNotImplemented() -} - // -------------------------- +// mock node store interface -// mock node client interface - -type mockNodeClient struct { +type mockNodeStore struct { nodes []v1.Node } -func (m *mockNodeClient) Create(node *v1.Node) (*v1.Node, error) { - return nil, errorNotImplemented() -} -func (m *mockNodeClient) Update(node *v1.Node) (*v1.Node, error) { - return nil, errorNotImplemented() -} -func (m *mockNodeClient) UpdateStatus(node *v1.Node) (*v1.Node, error) { - return nil, errorNotImplemented() -} -func (m *mockNodeClient) Delete(name string, options *metav1.DeleteOptions) error { - return errorNotImplemented() -} -func (m *mockNodeClient) Get(name string, options metav1.GetOptions) (*v1.Node, error) { - return nil, errorNotImplemented() -} -func (m *mockNodeClient) List(opts metav1.ListOptions) (*v1.NodeList, error) { - return &v1.NodeList{Items: m.nodes}, nil -} -func (m *mockNodeClient) Watch(opts metav1.ListOptions) (watch.Interface, error) { - return nil, errorNotImplemented() -} -func (m *mockNodeClient) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Node, err error) { - return nil, errorNotImplemented() +func (m *mockNodeStore) List(ls labels.Selector) ([]v1.Node, error) { + return m.nodes, nil } // -------------------------- - // utility functions func assertEqual(t *testing.T, a interface{}, b interface{}) { @@ -241,11 +241,6 @@ func errorAlreadyExists() error { return apierrors.NewAlreadyExists(schema.GroupResource{}, "already-exists") } -func errorNotImplemented() error { - log.Fatal("not implemented") - return apierrors.NewMethodNotSupported(schema.GroupResource{}, "not-implemented") -} - func logMemUsage(t *testing.T) { var stats runtime.MemStats runtime.ReadMemStats(&stats) diff --git a/pkg/server/router.go b/pkg/server/router.go index 8c05339d9a83..d1e112ecec6b 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -431,8 +431,8 @@ type nodeInfo struct { func passwordBootstrap(ctx context.Context, config *Config) nodePassBootstrapper { runtime := config.ControlConfig.Runtime deferredNodes := map[string]bool{} - var secretClient coreclient.SecretClient - var nodeClient coreclient.NodeClient + var secretClient coreclient.SecretController + var nodeClient coreclient.NodeController var mu sync.Mutex return nodePassBootstrapper(func(req *http.Request) (string, int, error) { @@ -535,9 +535,9 @@ func verifyRemotePassword(ctx context.Context, config *Config, mu *sync.Mutex, d return node.Name, http.StatusOK, nil } -func verifyNode(ctx context.Context, nodeClient coreclient.NodeClient, node *nodeInfo) error { +func verifyNode(ctx context.Context, nodeClient coreclient.NodeController, node *nodeInfo) error { if nodeName, isNodeAuth := identifier.NodeIdentity(node.User); isNodeAuth { - if _, err := nodeClient.Get(nodeName, metav1.GetOptions{}); err != nil { + if _, err := nodeClient.Cache().Get(nodeName); err != nil { return errors.Wrap(err, "unable to verify node identity") } } From 0731e80a9f33ccb4519e27b98630c5c33637d441 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 16 Nov 2023 00:24:09 +0000 Subject: [PATCH 10/10] Go generate Signed-off-by: Brad Davidson --- .../controllers/k3s.cattle.io/v1/addon.go | 97 +---------------- .../k3s.cattle.io/v1/etcdsnapshotfile.go | 103 +----------------- .../controllers/k3s.cattle.io/v1/interface.go | 8 +- 3 files changed, 7 insertions(+), 201 deletions(-) diff --git a/pkg/generated/controllers/k3s.cattle.io/v1/addon.go b/pkg/generated/controllers/k3s.cattle.io/v1/addon.go index 73418d9520fd..1b81631e2e67 100644 --- a/pkg/generated/controllers/k3s.cattle.io/v1/addon.go +++ b/pkg/generated/controllers/k3s.cattle.io/v1/addon.go @@ -19,114 +19,21 @@ limitations under the License. package v1 import ( - "context" - "time" - v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1" "github.com/rancher/wrangler/pkg/generic" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/watch" ) // AddonController interface for managing Addon resources. type AddonController interface { - generic.ControllerMeta - AddonClient - - // OnChange runs the given handler when the controller detects a resource was changed. - OnChange(ctx context.Context, name string, sync AddonHandler) - - // OnRemove runs the given handler when the controller detects a resource was changed. - OnRemove(ctx context.Context, name string, sync AddonHandler) - - // Enqueue adds the resource with the given name to the worker queue of the controller. - Enqueue(namespace, name string) - - // EnqueueAfter runs Enqueue after the provided duration. - EnqueueAfter(namespace, name string, duration time.Duration) - - // Cache returns a cache for the resource type T. - Cache() AddonCache + generic.ControllerInterface[*v1.Addon, *v1.AddonList] } // AddonClient interface for managing Addon resources in Kubernetes. type AddonClient interface { - // Create creates a new object and return the newly created Object or an error. - Create(*v1.Addon) (*v1.Addon, error) - - // Update updates the object and return the newly updated Object or an error. - Update(*v1.Addon) (*v1.Addon, error) - - // Delete deletes the Object in the given name. - Delete(namespace, name string, options *metav1.DeleteOptions) error - - // Get will attempt to retrieve the resource with the specified name. - Get(namespace, name string, options metav1.GetOptions) (*v1.Addon, error) - - // List will attempt to find multiple resources. - List(namespace string, opts metav1.ListOptions) (*v1.AddonList, error) - - // Watch will start watching resources. - Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error) - - // Patch will patch the resource with the matching name. - Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Addon, err error) + generic.ClientInterface[*v1.Addon, *v1.AddonList] } // AddonCache interface for retrieving Addon resources in memory. type AddonCache interface { - // Get returns the resources with the specified name from the cache. - Get(namespace, name string) (*v1.Addon, error) - - // List will attempt to find resources from the Cache. - List(namespace string, selector labels.Selector) ([]*v1.Addon, error) - - // AddIndexer adds a new Indexer to the cache with the provided name. - // If you call this after you already have data in the store, the results are undefined. - AddIndexer(indexName string, indexer AddonIndexer) - - // GetByIndex returns the stored objects whose set of indexed values - // for the named index includes the given indexed value. - GetByIndex(indexName, key string) ([]*v1.Addon, error) -} - -// AddonHandler is function for performing any potential modifications to a Addon resource. -type AddonHandler func(string, *v1.Addon) (*v1.Addon, error) - -// AddonIndexer computes a set of indexed values for the provided object. -type AddonIndexer func(obj *v1.Addon) ([]string, error) - -// AddonGenericController wraps wrangler/pkg/generic.Controller so that the function definitions adhere to AddonController interface. -type AddonGenericController struct { - generic.ControllerInterface[*v1.Addon, *v1.AddonList] -} - -// OnChange runs the given resource handler when the controller detects a resource was changed. -func (c *AddonGenericController) OnChange(ctx context.Context, name string, sync AddonHandler) { - c.ControllerInterface.OnChange(ctx, name, generic.ObjectHandler[*v1.Addon](sync)) -} - -// OnRemove runs the given object handler when the controller detects a resource was changed. -func (c *AddonGenericController) OnRemove(ctx context.Context, name string, sync AddonHandler) { - c.ControllerInterface.OnRemove(ctx, name, generic.ObjectHandler[*v1.Addon](sync)) -} - -// Cache returns a cache of resources in memory. -func (c *AddonGenericController) Cache() AddonCache { - return &AddonGenericCache{ - c.ControllerInterface.Cache(), - } -} - -// AddonGenericCache wraps wrangler/pkg/generic.Cache so the function definitions adhere to AddonCache interface. -type AddonGenericCache struct { generic.CacheInterface[*v1.Addon] } - -// AddIndexer adds a new Indexer to the cache with the provided name. -// If you call this after you already have data in the store, the results are undefined. -func (c AddonGenericCache) AddIndexer(indexName string, indexer AddonIndexer) { - c.CacheInterface.AddIndexer(indexName, generic.Indexer[*v1.Addon](indexer)) -} diff --git a/pkg/generated/controllers/k3s.cattle.io/v1/etcdsnapshotfile.go b/pkg/generated/controllers/k3s.cattle.io/v1/etcdsnapshotfile.go index ad9a1cdf3052..8358d1ae3b12 100644 --- a/pkg/generated/controllers/k3s.cattle.io/v1/etcdsnapshotfile.go +++ b/pkg/generated/controllers/k3s.cattle.io/v1/etcdsnapshotfile.go @@ -29,133 +29,36 @@ import ( "github.com/rancher/wrangler/pkg/kv" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/watch" ) // ETCDSnapshotFileController interface for managing ETCDSnapshotFile resources. type ETCDSnapshotFileController interface { - generic.ControllerMeta - ETCDSnapshotFileClient - - // OnChange runs the given handler when the controller detects a resource was changed. - OnChange(ctx context.Context, name string, sync ETCDSnapshotFileHandler) - - // OnRemove runs the given handler when the controller detects a resource was changed. - OnRemove(ctx context.Context, name string, sync ETCDSnapshotFileHandler) - - // Enqueue adds the resource with the given name to the worker queue of the controller. - Enqueue(name string) - - // EnqueueAfter runs Enqueue after the provided duration. - EnqueueAfter(name string, duration time.Duration) - - // Cache returns a cache for the resource type T. - Cache() ETCDSnapshotFileCache + generic.NonNamespacedControllerInterface[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList] } // ETCDSnapshotFileClient interface for managing ETCDSnapshotFile resources in Kubernetes. type ETCDSnapshotFileClient interface { - // Create creates a new object and return the newly created Object or an error. - Create(*v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) - - // Update updates the object and return the newly updated Object or an error. - Update(*v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) - // UpdateStatus updates the Status field of a the object and return the newly updated Object or an error. - // Will always return an error if the object does not have a status field. - UpdateStatus(*v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) - - // Delete deletes the Object in the given name. - Delete(name string, options *metav1.DeleteOptions) error - - // Get will attempt to retrieve the resource with the specified name. - Get(name string, options metav1.GetOptions) (*v1.ETCDSnapshotFile, error) - - // List will attempt to find multiple resources. - List(opts metav1.ListOptions) (*v1.ETCDSnapshotFileList, error) - - // Watch will start watching resources. - Watch(opts metav1.ListOptions) (watch.Interface, error) - - // Patch will patch the resource with the matching name. - Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.ETCDSnapshotFile, err error) + generic.NonNamespacedClientInterface[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList] } // ETCDSnapshotFileCache interface for retrieving ETCDSnapshotFile resources in memory. type ETCDSnapshotFileCache interface { - // Get returns the resources with the specified name from the cache. - Get(name string) (*v1.ETCDSnapshotFile, error) - - // List will attempt to find resources from the Cache. - List(selector labels.Selector) ([]*v1.ETCDSnapshotFile, error) - - // AddIndexer adds a new Indexer to the cache with the provided name. - // If you call this after you already have data in the store, the results are undefined. - AddIndexer(indexName string, indexer ETCDSnapshotFileIndexer) - - // GetByIndex returns the stored objects whose set of indexed values - // for the named index includes the given indexed value. - GetByIndex(indexName, key string) ([]*v1.ETCDSnapshotFile, error) -} - -// ETCDSnapshotFileHandler is function for performing any potential modifications to a ETCDSnapshotFile resource. -type ETCDSnapshotFileHandler func(string, *v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) - -// ETCDSnapshotFileIndexer computes a set of indexed values for the provided object. -type ETCDSnapshotFileIndexer func(obj *v1.ETCDSnapshotFile) ([]string, error) - -// ETCDSnapshotFileGenericController wraps wrangler/pkg/generic.NonNamespacedController so that the function definitions adhere to ETCDSnapshotFileController interface. -type ETCDSnapshotFileGenericController struct { - generic.NonNamespacedControllerInterface[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList] -} - -// OnChange runs the given resource handler when the controller detects a resource was changed. -func (c *ETCDSnapshotFileGenericController) OnChange(ctx context.Context, name string, sync ETCDSnapshotFileHandler) { - c.NonNamespacedControllerInterface.OnChange(ctx, name, generic.ObjectHandler[*v1.ETCDSnapshotFile](sync)) -} - -// OnRemove runs the given object handler when the controller detects a resource was changed. -func (c *ETCDSnapshotFileGenericController) OnRemove(ctx context.Context, name string, sync ETCDSnapshotFileHandler) { - c.NonNamespacedControllerInterface.OnRemove(ctx, name, generic.ObjectHandler[*v1.ETCDSnapshotFile](sync)) -} - -// Cache returns a cache of resources in memory. -func (c *ETCDSnapshotFileGenericController) Cache() ETCDSnapshotFileCache { - return &ETCDSnapshotFileGenericCache{ - c.NonNamespacedControllerInterface.Cache(), - } -} - -// ETCDSnapshotFileGenericCache wraps wrangler/pkg/generic.NonNamespacedCache so the function definitions adhere to ETCDSnapshotFileCache interface. -type ETCDSnapshotFileGenericCache struct { generic.NonNamespacedCacheInterface[*v1.ETCDSnapshotFile] } -// AddIndexer adds a new Indexer to the cache with the provided name. -// If you call this after you already have data in the store, the results are undefined. -func (c ETCDSnapshotFileGenericCache) AddIndexer(indexName string, indexer ETCDSnapshotFileIndexer) { - c.NonNamespacedCacheInterface.AddIndexer(indexName, generic.Indexer[*v1.ETCDSnapshotFile](indexer)) -} - type ETCDSnapshotFileStatusHandler func(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) (v1.ETCDSnapshotStatus, error) type ETCDSnapshotFileGeneratingHandler func(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) ([]runtime.Object, v1.ETCDSnapshotStatus, error) -func FromETCDSnapshotFileHandlerToHandler(sync ETCDSnapshotFileHandler) generic.Handler { - return generic.FromObjectHandlerToHandler(generic.ObjectHandler[*v1.ETCDSnapshotFile](sync)) -} - func RegisterETCDSnapshotFileStatusHandler(ctx context.Context, controller ETCDSnapshotFileController, condition condition.Cond, name string, handler ETCDSnapshotFileStatusHandler) { statusHandler := &eTCDSnapshotFileStatusHandler{ client: controller, condition: condition, handler: handler, } - controller.AddGenericHandler(ctx, name, FromETCDSnapshotFileHandlerToHandler(statusHandler.sync)) + controller.AddGenericHandler(ctx, name, generic.FromObjectHandlerToHandler(statusHandler.sync)) } func RegisterETCDSnapshotFileGeneratingHandler(ctx context.Context, controller ETCDSnapshotFileController, apply apply.Apply, diff --git a/pkg/generated/controllers/k3s.cattle.io/v1/interface.go b/pkg/generated/controllers/k3s.cattle.io/v1/interface.go index ba85bd1e850d..6a80c591acb4 100644 --- a/pkg/generated/controllers/k3s.cattle.io/v1/interface.go +++ b/pkg/generated/controllers/k3s.cattle.io/v1/interface.go @@ -46,13 +46,9 @@ type version struct { } func (v *version) Addon() AddonController { - return &AddonGenericController{ - generic.NewController[*v1.Addon, *v1.AddonList](schema.GroupVersionKind{Group: "k3s.cattle.io", Version: "v1", Kind: "Addon"}, "addons", true, v.controllerFactory), - } + return generic.NewController[*v1.Addon, *v1.AddonList](schema.GroupVersionKind{Group: "k3s.cattle.io", Version: "v1", Kind: "Addon"}, "addons", true, v.controllerFactory) } func (v *version) ETCDSnapshotFile() ETCDSnapshotFileController { - return &ETCDSnapshotFileGenericController{ - generic.NewNonNamespacedController[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList](schema.GroupVersionKind{Group: "k3s.cattle.io", Version: "v1", Kind: "ETCDSnapshotFile"}, "etcdsnapshotfiles", v.controllerFactory), - } + return generic.NewNonNamespacedController[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList](schema.GroupVersionKind{Group: "k3s.cattle.io", Version: "v1", Kind: "ETCDSnapshotFile"}, "etcdsnapshotfiles", v.controllerFactory) }