[release-1.29] Backports for 2025-01

go.mod

@@ -19,7 +19,7 @@ replace (
 	github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.12.0
 	github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.29.0-k3s1
 	github.com/open-policy-agent/opa => github.com/open-policy-agent/opa v0.59.0 // github.com/Microsoft/hcsshim using bad version v0.42.2
-	github.com/opencontainers/runc => github.com/k3s-io/runc v1.2.1-k3s1
+	github.com/opencontainers/runc => github.com/k3s-io/runc v1.2.4-k3s1
 	github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.11.0
 	github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common => github.com/prometheus/common v0.45.0
@@ -149,7 +149,7 @@ require (
 	golang.org/x/crypto v0.27.0
 	golang.org/x/net v0.29.0
 	golang.org/x/sync v0.8.0
-	golang.org/x/sys v0.25.0
+	golang.org/x/sys v0.28.0
 	google.golang.org/grpc v1.68.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.30.3
@@ -231,7 +231,7 @@ require (
 	github.com/coreos/go-oidc v2.2.1+incompatible // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/cyphar/filepath-securejoin v0.3.4 // indirect
+	github.com/cyphar/filepath-securejoin v0.3.5 // indirect
 	github.com/danwinship/knftables v0.0.13 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/daviddengcn/go-colortext v1.0.0 // indirect

go.sum

@@ -493,8 +493,8 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8=
-github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM=
+github.com/cyphar/filepath-securejoin v0.3.5 h1:L81NHjquoQmcPgXcttUS9qTSR/+bXry6pbSINQGpjj4=
+github.com/cyphar/filepath-securejoin v0.3.5/go.mod h1:edhVd3c6OXKjUmSrVa/tGJRS9joFTxlslFCAyaxigkE=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
@@ -1063,8 +1063,8 @@ github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.29.12-k3s1 h1:xvb
 github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.29.12-k3s1/go.mod h1:4KmkE88Y4LDYrotr6iqMrolXDcWWY7UqmroXTO/sxFw=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.29.12-k3s1 h1:1FNNr2K5UzbPxP6bDhQAFZ1JOR2Xx9PCGxwm566HmmA=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.29.12-k3s1/go.mod h1:3nvUgy9DAoVbLCBJcIBDCldv+vAc7hcHl6xJFRSpvb4=
-github.com/k3s-io/runc v1.2.1-k3s1 h1:74ZffmoNVdX+jO+XYLv0iU/A9Yse1DZ2slZog7gNDgE=
-github.com/k3s-io/runc v1.2.1-k3s1/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc=
+github.com/k3s-io/runc v1.2.4-k3s1 h1:wCqlmxJiVUsZvdNsSUUm7Ng0kCK2wAgyBt4lcp2lLP8=
+github.com/k3s-io/runc v1.2.4-k3s1/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM=
 github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314 h1:TrZb/yM0OtBuifPXlKaOfcxpJqzakA8+KsoO4c69ZLM=
 github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314/go.mod h1:bMHfSjj1+Zf5VITCZe/wLjuni6rYAj/DjPU/kIVnhfA=
 github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=

manifests/rolebindings.yaml

@@ -23,6 +23,14 @@ rules:
   - nodes
   verbs:
   - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
 - apiGroups:
   - ""
   resources:

manifests/traefik.yaml

@@ -28,7 +28,7 @@ spec:
     priorityClassName: "system-cluster-critical"
     image:
       repository: "rancher/mirrored-library-traefik"
-      tag: "2.11.10"
+      tag: "2.11.18"
     tolerations:
     - key: "CriticalAddonsOnly"
       operator: "Exists"

pkg/agent/config/config.go

@@ -2,9 +2,11 @@ package config
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	cryptorand "crypto/rand"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/hex"
 	"encoding/pem"
 	"fmt"
@@ -32,6 +34,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
 	"github.com/pkg/errors"
+	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/rancher/wrangler/pkg/slice"
 	"github.com/sirupsen/logrus"
@@ -133,9 +136,9 @@ func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]b
 	return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts, info.CertFile, info.KeyFile), info.Username, info.Password, info.Token())
 }
 
-func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string) HTTPRequester {
+func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string, csr []byte) HTTPRequester {
 	return func(u string, client *http.Client, username, password, token string) ([]byte, error) {
-		req, err := http.NewRequest(http.MethodGet, u, nil)
+		req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(csr))
 		if err != nil {
 			return nil, err
 		}
@@ -238,51 +241,93 @@ func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string)
 	}
 }
 
-func getServingCert(nodeName string, nodeIPs []net.IP, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
-	servingCert, err := Request("/v1-"+version.Program+"/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
+// getKubeletServingCert fills the kubelet server certificate with content returned
+// from the server.  We attempt to POST a CSR to the server, in hopes that it will
+// sign the cert using our locally generated key. If the server does not support CSR
+// signing, the key generated by the server is used instead.
+func getKubeletServingCert(nodeName string, nodeIPs []net.IP, certFile, keyFile, nodePasswordFile string, info *clientaccess.Info) error {
+	csr, err := getCSRBytes(keyFile)
 	if err != nil {
-		return nil, err
+		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
 	}
 
-	servingCert, servingKey := splitCertKeyPEM(servingCert)
-
-	if err := os.WriteFile(servingCertFile, servingCert, 0600); err != nil {
-		return nil, errors.Wrapf(err, "failed to write node cert")
+	basename := filepath.Base(certFile)
+	body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
+	if err != nil {
+		return err
 	}
 
-	if err := os.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
-		return nil, errors.Wrapf(err, "failed to write node key")
+	// Always split the response, as down-level servers may send back a cert+key
+	// instead of signing a new cert with our key.  If the response includes a key it
+	// must be used instead of the one we signed the CSR with.
+	certBytes, keyBytes := splitCertKeyPEM(body)
+	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", certFile)
 	}
-
-	cert, err := tls.X509KeyPair(servingCert, servingKey)
-	if err != nil {
-		return nil, err
+	if len(keyBytes) > 0 {
+		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
+			return errors.Wrapf(err, "failed to write key %s", keyFile)
+		}
 	}
-	return &cert, nil
+	return nil
 }
 
-func getHostFile(filename, keyFile string, info *clientaccess.Info) error {
+// getHostFile fills a file with content returned from the server.
+func getHostFile(filename string, info *clientaccess.Info) error {
 	basename := filepath.Base(filename)
 	fileBytes, err := info.Get("/v1-" + version.Program + "/" + basename)
 	if err != nil {
 		return err
 	}
-	if keyFile == "" {
-		if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write cert %s", filename)
-		}
-	} else {
-		fileBytes, keyBytes := splitCertKeyPEM(fileBytes)
-		if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write cert %s", filename)
-		}
+	if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", filename)
+	}
+	return nil
+}
+
+// getClientCert fills a client certificate with content returned from the server.
+// We attempt to POST a CSR to the server, in hopes that it will sign the cert using
+// our locally generated key. If the server does not support CSR signing, the key
+// generated by the server is used instead.
+func getClientCert(certFile, keyFile string, info *clientaccess.Info) error {
+	csr, err := getCSRBytes(keyFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
+	}
+
+	basename := filepath.Base(certFile)
+	fileBytes, err := info.Post("/v1-"+version.Program+"/"+basename, csr)
+	if err != nil {
+		return err
+	}
+
+	// Always split the response, as down-level servers may send back a cert+key
+	// instead of signing a new cert with our key.  If the response includes a key it
+	// must be used instead of the one we signed the CSR with.
+	certBytes, keyBytes := splitCertKeyPEM(fileBytes)
+	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", certFile)
+	}
+	if len(keyBytes) > 0 {
 		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write key %s", filename)
+			return errors.Wrapf(err, "failed to write key %s", keyFile)
 		}
 	}
 	return nil
 }
 
+func getCSRBytes(keyFile string) ([]byte, error) {
+	keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false)
+	if err != nil {
+		return nil, err
+	}
+	key, err := certutil.ParsePrivateKeyPEM(keyBytes)
+	if err != nil {
+		return nil, err
+	}
+	return x509.CreateCertificateRequest(cryptorand.Reader, &x509.CertificateRequest{}, key)
+}
+
 func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
 	for {
 		b, rest := pem.Decode(bytes)
@@ -301,19 +346,33 @@ func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
 	return
 }
 
-func getNodeNamedHostFile(filename, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
-	basename := filepath.Base(filename)
-	fileBytes, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
+// getKubeletClientCert fills the kubelet client certificate with content returned
+// from the server.  We attempt to POST a CSR to the server, in hopes that it will
+// sign the cert using our locally generated key. If the server does not support CSR
+// signing, the key generated by the server is used instead.
+func getKubeletClientCert(certFile, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
+	csr, err := getCSRBytes(keyFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
+	}
+
+	basename := filepath.Base(certFile)
+	body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
 	if err != nil {
 		return err
 	}
-	fileBytes, keyBytes := splitCertKeyPEM(fileBytes)
 
-	if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write cert %s", filename)
+	// Always split the response, as down-level servers may send back a cert+key
+	// instead of signing a new cert with our key.  If the response includes a key it
+	// must be used instead of the one we signed the CSR with.
+	certBytes, keyBytes := splitCertKeyPEM(body)
+	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", certFile)
 	}
-	if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write key %s", filename)
+	if len(keyBytes) > 0 {
+		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
+			return errors.Wrapf(err, "failed to write key %s", keyFile)
+		}
 	}
 	return nil
 }
@@ -399,12 +458,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 
 	clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt")
-	if err := getHostFile(clientCAFile, "", info); err != nil {
+	if err := getHostFile(clientCAFile, info); err != nil {
 		return nil, err
 	}
 
 	serverCAFile := filepath.Join(envInfo.DataDir, "agent", "server-ca.crt")
-	if err := getHostFile(serverCAFile, "", info); err != nil {
+	if err := getHostFile(serverCAFile, info); err != nil {
 		return nil, err
 	}
 
@@ -498,14 +557,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	// that the cert will not be valid for, as they are not present in the list collected here.
 	nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...)
 
-	// Ask the server to generate a kubelet server cert+key. These files are unique to this node.
-	servingCert, err := getServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info)
-	if err != nil {
+	// Ask the server to sign our kubelet server cert.
+	if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
 		return nil, errors.Wrap(err, servingKubeletCert)
 	}
 
-	// Ask the server to genrate a kubelet client cert+key. These files are unique to this node.
-	if err := getNodeNamedHostFile(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
+	// Ask the server to sign our kubelet client cert.
+	if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
 		return nil, errors.Wrap(err, clientKubeletCert)
 	}
 
@@ -518,8 +576,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt")
 	clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key")
 
-	// Ask the server to send us its kube-proxy client cert+key. These files are not unique to this node.
-	if err := getHostFile(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
+	// Ask the server to sign our kube-proxy client cert.
+	if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
 		return nil, errors.Wrap(err, clientKubeProxyCert)
 	}
 
@@ -532,8 +590,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt")
 	clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key")
 
-	// Ask the server to send us its agent controller client cert+key. These files are not unique to this node.
-	if err := getHostFile(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
+	// Ask the server to sign our agent controller client cert.
+	if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
 		return nil, errors.Wrap(err, clientK3sControllerCert)
 	}
 
@@ -618,7 +676,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	applyCRIDockerdAddress(nodeConfig)
 	applyContainerdQoSClassConfigFileIfPresent(envInfo, &nodeConfig.Containerd)
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl")
-	nodeConfig.Certificate = servingCert
 
 	if envInfo.BindAddress != "" {
 		nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress