test: add kwok e2e environment

test/README.md

@@ -0,0 +1,10 @@
+# E2E Testing
+
+Karpenter leverages Github Actions to run our E2E test suites. 
+
+## Directories
+- `./.github/workflows`: Workflow files run within this repository. Relevant files for E2E testing are prefixed with `e2e-`
+- `./.github/actions/e2e`: Composite actions utilized by the E2E workflows
+- `./test/suites`: Directories defining test suites
+- `./test/pkg`: Common utilities and expectations
+- `./test/hack`: Testing scripts

test/pkg/debug/events.go

@@ -0,0 +1,135 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package debug
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/samber/lo"
+	"go.uber.org/multierr"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type EventClient struct {
+	start      time.Time
+	kubeClient client.Client
+}
+
+func NewEventClient(kubeClient client.Client) *EventClient {
+	return &EventClient{
+		start:      time.Now(),
+		kubeClient: kubeClient,
+	}
+}
+
+func (c *EventClient) DumpEvents(ctx context.Context) error {
+	return multierr.Combine(
+		c.dumpPodEvents(ctx),
+		c.dumpNodeEvents(ctx),
+	)
+
+}
+
+func (c *EventClient) dumpPodEvents(ctx context.Context) error {
+	el := &v1.EventList{}
+	if err := c.kubeClient.List(ctx, el, &client.ListOptions{
+		FieldSelector: fields.SelectorFromSet(map[string]string{"involvedObject.kind": "Pod"}),
+	}); err != nil {
+		return err
+	}
+	events := lo.Filter(filterTestEvents(el.Items, c.start), func(e v1.Event, _ int) bool {
+		return e.InvolvedObject.Namespace != "kube-system"
+	})
+	for k, v := range coallateEvents(events) {
+		fmt.Print(getEventInformation(k, v))
+	}
+	return nil
+}
+
+func (c *EventClient) dumpNodeEvents(ctx context.Context) error {
+	el := &v1.EventList{}
+	if err := c.kubeClient.List(ctx, el, &client.ListOptions{
+		FieldSelector: fields.SelectorFromSet(map[string]string{"involvedObject.kind": "Node"}),
+	}); err != nil {
+		return err
+	}
+	for k, v := range coallateEvents(filterTestEvents(el.Items, c.start)) {
+		fmt.Print(getEventInformation(k, v))
+	}
+	return nil
+}
+
+func filterTestEvents(events []v1.Event, startTime time.Time) []v1.Event {
+	return lo.Filter(events, func(e v1.Event, _ int) bool {
+		if !e.EventTime.IsZero() {
+			if e.EventTime.BeforeTime(&metav1.Time{Time: startTime}) {
+				return false
+			}
+		} else if e.FirstTimestamp.Before(&metav1.Time{Time: startTime}) {
+			return false
+		}
+		return true
+	})
+}
+
+func coallateEvents(events []v1.Event) map[v1.ObjectReference]*v1.EventList {
+	eventMap := map[v1.ObjectReference]*v1.EventList{}
+	for i := range events {
+		elem := events[i]
+		objectKey := v1.ObjectReference{Kind: elem.InvolvedObject.Kind, Namespace: elem.InvolvedObject.Namespace, Name: elem.InvolvedObject.Name}
+		if _, ok := eventMap[objectKey]; !ok {
+			eventMap[objectKey] = &v1.EventList{}
+		}
+		eventMap[objectKey].Items = append(eventMap[objectKey].Items, elem)
+	}
+	return eventMap
+}
+
+// Partially copied from
+// https://github.com/kubernetes/kubernetes/blob/04ee339c7a4d36b4037ce3635993e2a9e395ebf3/staging/src/k8s.io/kubectl/pkg/describe/describe.go#L4232
+func getEventInformation(o v1.ObjectReference, el *v1.EventList) string {
+	sb := strings.Builder{}
+	sb.WriteString(fmt.Sprintf("------- %s/%s%s EVENTS -------\n",
+		strings.ToLower(o.Kind), lo.Ternary(o.Namespace != "", o.Namespace+"/", ""), o.Name))
+	if len(el.Items) == 0 {
+		return sb.String()
+	}
+	for _, e := range el.Items {
+		source := e.Source.Component
+		if source == "" {
+			source = e.ReportingController
+		}
+		eventTime := e.EventTime
+		if eventTime.IsZero() {
+			eventTime = metav1.NewMicroTime(e.FirstTimestamp.Time)
+		}
+		sb.WriteString(fmt.Sprintf("time=%s type=%s reason=%s from=%s message=%s\n",
+			eventTime.Format(time.RFC3339),
+			e.Type,
+			e.Reason,
+			source,
+			strings.TrimSpace(e.Message)),
+		)
+	}
+	return sb.String()
+}

test/pkg/debug/monitor.go

@@ -0,0 +1,83 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package debug
+
+import (
+	"context"
+	"sync"
+
+	"github.com/samber/lo"
+	"k8s.io/client-go/rest"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+	"sigs.k8s.io/karpenter/pkg/operator/controller"
+
+	"sigs.k8s.io/karpenter/pkg/operator/scheme"
+)
+
+type Monitor struct {
+	ctx    context.Context
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
+	mgr    manager.Manager
+}
+
+func New(ctx context.Context, config *rest.Config, kubeClient client.Client) *Monitor {
+	log.SetLogger(log.FromContext(ctx))
+	mgr := lo.Must(controllerruntime.NewManager(config, controllerruntime.Options{
+		Scheme: scheme.Scheme,
+		Metrics: server.Options{
+			BindAddress: "0",
+		},
+	}))
+	for _, c := range newControllers(kubeClient) {
+		lo.Must0(c.Register(ctx, mgr), "failed to register controller")
+	}
+	ctx, cancel := context.WithCancel(ctx) // this context is only meant for monitor start/stop
+	return &Monitor{
+		ctx:    ctx,
+		cancel: cancel,
+		mgr:    mgr,
+	}
+}
+
+// MustStart starts the debug monitor
+func (m *Monitor) MustStart() {
+	m.wg.Add(1)
+	go func() {
+		defer m.wg.Done()
+		lo.Must0(m.mgr.Start(m.ctx))
+	}()
+}
+
+// Stop stops the monitor
+func (m *Monitor) Stop() {
+	m.cancel()
+	m.wg.Wait()
+}
+
+func newControllers(kubeClient client.Client) []controller.Controller {
+	return []controller.Controller{
+		NewNodeClaimController(kubeClient),
+		NewNodeController(kubeClient),
+		NewPodController(kubeClient),
+	}
+}

test/pkg/debug/node.go

@@ -0,0 +1,84 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package debug
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
+
+	nodeutils "sigs.k8s.io/karpenter/pkg/utils/node"
+)
+
+type NodeController struct {
+	kubeClient client.Client
+}
+
+func NewNodeController(kubeClient client.Client) *NodeController {
+	return &NodeController{
+		kubeClient: kubeClient,
+	}
+}
+
+func (c *NodeController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	n := &v1.Node{}
+	if err := c.kubeClient.Get(ctx, req.NamespacedName, n); err != nil {
+		if errors.IsNotFound(err) {
+			fmt.Printf("[DELETED %s] NODE %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String())
+		}
+		return reconcile.Result{}, client.IgnoreNotFound(err)
+	}
+	fmt.Printf("[CREATED/UPDATED %s] NODE %s %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.Name, c.GetInfo(ctx, n))
+	return reconcile.Result{}, nil
+}
+
+func (c *NodeController) GetInfo(ctx context.Context, n *v1.Node) string {
+	pods, _ := nodeutils.GetPods(ctx, c.kubeClient, n)
+	return fmt.Sprintf("ready=%s schedulable=%t initialized=%s pods=%d taints=%v", nodeutils.GetCondition(n, v1.NodeReady).Status, !n.Spec.Unschedulable, n.Labels[v1beta1.NodeInitializedLabelKey], len(pods), n.Spec.Taints)
+}
+
+func (c *NodeController) Register(ctx context.Context, m manager.Manager) error {
+	return controllerruntime.NewControllerManagedBy(m).
+		Named("node").
+		For(&v1.Node{}).
+		WithEventFilter(predicate.And(
+			predicate.Funcs{
+				UpdateFunc: func(e event.UpdateEvent) bool {
+					oldNode := e.ObjectOld.(*v1.Node)
+					newNode := e.ObjectNew.(*v1.Node)
+					return c.GetInfo(ctx, oldNode) != c.GetInfo(ctx, newNode)
+				},
+			},
+			predicate.NewPredicateFuncs(func(o client.Object) bool {
+				return o.GetLabels()[v1beta1.NodePoolLabelKey] != ""
+			}),
+		)).
+		WithOptions(controller.Options{MaxConcurrentReconciles: 10}).
+		Complete(c)
+}

test/pkg/debug/nodeclaim.go

@@ -0,0 +1,80 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package debug
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
+)
+
+type NodeClaimController struct {
+	kubeClient client.Client
+}
+
+func NewNodeClaimController(kubeClient client.Client) *NodeClaimController {
+	return &NodeClaimController{
+		kubeClient: kubeClient,
+	}
+}
+
+func (c *NodeClaimController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	nc := &v1beta1.NodeClaim{}
+	if err := c.kubeClient.Get(ctx, req.NamespacedName, nc); err != nil {
+		if errors.IsNotFound(err) {
+			fmt.Printf("[DELETED %s] NODECLAIM %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String())
+		}
+		return reconcile.Result{}, client.IgnoreNotFound(err)
+	}
+	fmt.Printf("[CREATED/UPDATED %s] NODECLAIM %s %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.Name, c.GetInfo(nc))
+	return reconcile.Result{}, nil
+}
+
+func (c *NodeClaimController) GetInfo(nc *v1beta1.NodeClaim) string {
+	return fmt.Sprintf("ready=%t launched=%t registered=%t initialized=%t",
+		nc.StatusConditions().Root().IsTrue(),
+		nc.StatusConditions().Get(v1beta1.ConditionTypeLaunched).IsTrue(),
+		nc.StatusConditions().Get(v1beta1.ConditionTypeRegistered).IsTrue(),
+		nc.StatusConditions().Get(v1beta1.ConditionTypeInitialized).IsTrue(),
+	)
+}
+
+func (c *NodeClaimController) Register(_ context.Context, m manager.Manager) error {
+	return controllerruntime.NewControllerManagedBy(m).
+		Named("nodeclaim").
+		For(&v1beta1.NodeClaim{}).
+		WithEventFilter(predicate.Funcs{
+			UpdateFunc: func(e event.UpdateEvent) bool {
+				oldNodeClaim := e.ObjectOld.(*v1beta1.NodeClaim)
+				newNodeClaim := e.ObjectNew.(*v1beta1.NodeClaim)
+				return c.GetInfo(oldNodeClaim) != c.GetInfo(newNodeClaim)
+			},
+		}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: 10}).
+		Complete(c)
+}