diff --git a/.gitignore b/.gitignore index 459ff0ba0a8b..942c8493754a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ coverage.html *.test *.cpuprofile *.heapprofile +*.swp go.work go.work.sum diff --git a/Makefile b/Makefile index 45a245f75311..4418336ecc95 100644 --- a/Makefile +++ b/Makefile @@ -126,7 +126,7 @@ licenses: download ## Verifies dependency licenses ! go-licenses csv ./... | grep -v -e 'MIT' -e 'Apache-2.0' -e 'BSD-3-Clause' -e 'BSD-2-Clause' -e 'ISC' -e 'MPL-2.0' -e 'github.com/awslabs/amazon-eks-ami/nodeadm' image: ## Build the Karpenter controller images using ko build - $(eval CONTROLLER_IMG=$(shell $(WITH_GOFLAGS) KOCACHE=$(KOCACHE) KO_DOCKER_REPO="$(KO_DOCKER_REPO)" ko build --bare github.com/aws/karpenter-provider-aws/cmd/controller)) + $(eval CONTROLLER_IMG=$(shell $(WITH_GOFLAGS) KOCACHE=$(KOCACHE) KO_DOCKER_REPO="$(KO_DOCKER_REPO)" ko build --platform linux/amd64 --bare github.com/aws/karpenter-provider-aws/cmd/controller)) $(eval IMG_REPOSITORY=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 1)) $(eval IMG_TAG=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 2 -s)) $(eval IMG_DIGEST=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 2)) diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml index 0fa64cd6f2ec..c3d5a3bf84c2 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -155,6 +155,13 @@ spec: - WhenEmpty - WhenEmptyOrUnderutilized type: string + utilizationThreshold: + description: |- + UtilizationThreshold is defined as sum of requested resources divided by capacity + below which a node can be considered for disruption. + maximum: 100 + minimum: 1 + type: integer required: - consolidateAfter type: object diff --git a/go.mod b/go.mod index 00f4cef03640..9587f6f69f30 100644 --- a/go.mod +++ b/go.mod @@ -116,3 +116,5 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) + +replace sigs.k8s.io/karpenter v1.0.5 => github.com/smartnews/karpenter v1.0.5-sn-1 diff --git a/go.sum b/go.sum index ef5fe56a983a..6577f5393007 100644 --- a/go.sum +++ b/go.sum @@ -337,6 +337,8 @@ github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartnews/karpenter v1.0.5-sn-1 h1:+2ZOYrfE+0Acjct/7ERwrENlNqCso8Ht9i6pim/mnE8= +github.com/smartnews/karpenter v1.0.5-sn-1/go.mod h1:3NLmsnHHw8p4VutpjTOPUZyhE3qH6yGTs8O94Lsu8uw= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -761,8 +763,6 @@ sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHv sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v1.0.5 h1:QePds7w1dGCzTXI59fKpYyV5GL/LQgodOCMC1pYjqhM= -sigs.k8s.io/karpenter v1.0.5/go.mod h1:3NLmsnHHw8p4VutpjTOPUZyhE3qH6yGTs8O94Lsu8uw= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index bd1ff970467a..3d7a9f0678f0 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -155,6 +155,13 @@ spec: - WhenEmpty - WhenEmptyOrUnderutilized type: string + utilizationThreshold: + description: |- + UtilizationThreshold is defined as sum of requested resources divided by capacity + below which a node can be considered for disruption. + maximum: 100 + minimum: 1 + type: integer required: - consolidateAfter type: object diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 8f8c98d48c7b..5cd6d35f23af 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -23,7 +23,7 @@ const ( // resources. Cache hits enable faster provisioning and reduced API load on // AWS APIs, which can have a serious impact on performance and scalability. // DO NOT CHANGE THIS VALUE WITHOUT DUE CONSIDERATION - DefaultTTL = time.Minute + DefaultTTL = 5 * time.Minute // UnavailableOfferingsTTL is the time before offerings that were marked as unavailable // are removed from the cache and are available for launch again UnavailableOfferingsTTL = 3 * time.Minute diff --git a/pkg/controllers/interruption/controller.go b/pkg/controllers/interruption/controller.go index 731e3a741ac2..56598d8b3880 100644 --- a/pkg/controllers/interruption/controller.go +++ b/pkg/controllers/interruption/controller.go @@ -26,6 +26,7 @@ import ( "github.com/awslabs/operatorpkg/singleton" "go.uber.org/multierr" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" "k8s.io/utils/clock" @@ -198,6 +199,15 @@ func (c *Controller) handleNodeClaim(ctx context.Context, msg messages.Message, if zone != "" && instanceType != "" { c.unavailableOfferingsCache.MarkUnavailable(ctx, string(msg.Kind()), instanceType, zone, karpv1.CapacityTypeSpot) } + spotIntTotal.WithLabelValues(instanceType, zone, nodeClaim.Status.NodeName, nodeClaim.Labels["karpenter.sh/nodepool"]).Inc() + // try to create a new nodeclaim immediately but ignore error if it fails + if err := c.createNodeClaim(ctx, nodeClaim); err != nil { + log.FromContext(ctx).Error(err, "[interruption handling]failed to create a new nodeclaim") + } else { + log.FromContext(ctx).Info("Created new nodeclaim due to spot interruption") + // wait for the node provisioning before draining + time.Sleep(60 * time.Second) + } } if action != NoAction { return c.deleteNodeClaim(ctx, msg, nodeClaim, node) @@ -205,6 +215,20 @@ func (c *Controller) handleNodeClaim(ctx context.Context, msg messages.Message, return nil } +// createNodeClaim creates a new NodeClaim with the same spec of the interrupted one +func (c *Controller) createNodeClaim(ctx context.Context, oldNodeClaim *karpv1.NodeClaim) error { + newNodeClaim := &karpv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: oldNodeClaim.ObjectMeta.GenerateName, + Annotations: oldNodeClaim.ObjectMeta.Annotations, + Labels: oldNodeClaim.ObjectMeta.Labels, + OwnerReferences: oldNodeClaim.ObjectMeta.OwnerReferences, + }, + Spec: oldNodeClaim.Spec, + } + return c.kubeClient.Create(ctx, newNodeClaim) +} + // deleteNodeClaim removes the NodeClaim from the api-server func (c *Controller) deleteNodeClaim(ctx context.Context, msg messages.Message, nodeClaim *karpv1.NodeClaim, node *corev1.Node) error { if !nodeClaim.DeletionTimestamp.IsZero() { diff --git a/pkg/controllers/interruption/metrics.go b/pkg/controllers/interruption/metrics.go index 7b3f92468751..2de752ef8a62 100644 --- a/pkg/controllers/interruption/metrics.go +++ b/pkg/controllers/interruption/metrics.go @@ -24,6 +24,10 @@ import ( const ( interruptionSubsystem = "interruption" messageTypeLabel = "message_type" + instanceTypeLabel = "instance_type" + zoneLabel = "zone" + hostLabel = "node_name" + poolLabel = "node_pool" ) var ( @@ -53,8 +57,17 @@ var ( Buckets: metrics.DurationBuckets(), }, ) + spotIntTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metrics.Namespace, + Subsystem: interruptionSubsystem, + Name: "spot_int_total", + Help: "Number of the spot interruption. Labeled by AZ, instance type", + }, + []string{instanceTypeLabel, zoneLabel, hostLabel, poolLabel}, + ) ) func init() { - crmetrics.Registry.MustRegister(receivedMessages, deletedMessages, messageLatency) + crmetrics.Registry.MustRegister(receivedMessages, deletedMessages, messageLatency, spotIntTotal) } diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 4e37edaa7d73..92fca88c0933 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -264,6 +264,7 @@ func getTags(ctx context.Context, nodeClass *v1.EC2NodeClass, nodeClaim *karpv1. karpv1.NodePoolLabelKey: nodeClaim.Labels[karpv1.NodePoolLabelKey], v1.EKSClusterNameTagKey: options.FromContext(ctx).ClusterName, v1.LabelNodeClass: nodeClass.Name, + "Component": nodeClaim.Labels[karpv1.NodePoolLabelKey], // used for aws explore } return lo.Assign(nodeClass.Spec.Tags, staticTags) }