Apply review comments. Add debug support (#7)

* Apply review comments. Add debug support

* Apply review comments

* Metrics client: cap input stream length

* Development README cosmetics

Co-authored-by: Ismail Alidzhikov <[email protected]>

* README cosmetics

Co-authored-by: Ismail Alidzhikov <[email protected]>

* Makefile - remove TODO

* Bump pipeline_definitions to golang:1.22.1

* Drop inactive configuration from skaffold.yaml

---------

Co-authored-by: Ismail Alidzhikov <[email protected]>

.ci/pipeline_definitions

@@ -36,7 +36,7 @@ gardener-custom-metrics:
     release:
       steps:
         verify:
-          image: 'golang:1.19.9'
+          image: 'golang:1.22.1'
       traits:
         version:
           preprocess: 'finalize'

Dockerfile

@@ -1,18 +1,16 @@
 ############# builder
-FROM golang:1.19.9 AS builder
+FROM golang:1.22.1 AS builder
 
 WORKDIR /go/src/github.com/gardener/gardener-custom-metrics
 COPY . .
 RUN make install
-# RUN CGO_ENABLED=0 GO111MODULE=on GOFLAGS=-mod=vendor go build -a -o gardener-custom-metrics.exe cmd/main.go
 
 ############# base image # TODO: Andrey: P1: Move to distroless
-FROM alpine:3.18.0 AS base
+FROM alpine:3.18.6 AS base
 
 ############# gardener-custom-metrics
 FROM base AS gardener-custom-metrics
 WORKDIR /
 
-COPY --from=builder /go/bin/gardener-custom-metrics /gardener-custom-metrics.exe
-# COPY --from=builder /go/src/github.com/gardener/gardener-custom-metrics/gardener-custom-metrics.exe .
-ENTRYPOINT ["/gardener-custom-metrics.exe"]
+COPY --from=builder /go/bin/gardener-custom-metrics /gardener-custom-metrics
+ENTRYPOINT ["/gardener-custom-metrics"]

Makefile

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+BUILD_DATE := $(shell date '+%Y-%m-%dT%H:%M:%S%z' | sed 's/\([0-9][0-9]\)$$/:\1/g')
 NAME                        := gardener-custom-metrics
-# IMAGE_REGISTRY_URI          := eu.gcr.io/gardener-project/gardener/$(NAME)
-IMAGE_REGISTRY_URI          := eu.gcr.io/sap-se-gcp-scp-k8s/$(NAME)
+IMAGE_REGISTRY_URI          := eu.gcr.io/gardener-project/gardener
 REPO_ROOT                   := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 VERSION                     := $(shell cat "$(REPO_ROOT)/VERSION")
 EFFECTIVE_VERSION           := $(VERSION)-$(shell git rev-parse HEAD)
@@ -30,7 +30,9 @@ ifneq ($(strip $(shell git status --porcelain 2>/dev/null)),)
 	EFFECTIVE_VERSION := $(EFFECTIVE_VERSION)-dirty
 endif
 
+# In debug, do not use the -w flag. It strips useful debug information.
 LD_FLAGS := "-w $(shell EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) $(REPO_ROOT)/hack/gardener-util/get-build-ld-flags.sh k8s.io/component-base $(REPO_ROOT)/VERSION $(EXTENSION_PREFIX)-$(NAME))"
+LD_FLAGS_DEBUG := "$(shell EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) $(REPO_ROOT)/hack/gardener-util/get-build-ld-flags.sh k8s.io/component-base $(REPO_ROOT)/VERSION $(EXTENSION_PREFIX)-$(NAME))"
 
 TOOLS_DIR := $(REPO_ROOT)/hack/gardener-util/tools
 include $(REPO_ROOT)/hack/gardener-util/tools.mk
@@ -136,3 +138,19 @@ verify: check check-docforge format test
 
 .PHONY: verify-extended
 verify-extended: check-generate check check-docforge format test test-cov test-clean
+
+# skaffold dev and debug clean up deployed modules by default, disable this
+debug: export SKAFFOLD_CLEANUP = false
+# Artifacts might be already built when you decide to start debugging.
+# However, these artifacts do not include the gcflags which `skaffold debug` sets automatically, so delve would not work.
+# Disabling the skaffold cache for debugging ensures that you run artifacts with gcflags required for debugging.
+debug: export SKAFFOLD_CACHE_ARTIFACTS = false
+
+debug: export SOURCE_DATE_EPOCH = $(shell date -d $(BUILD_DATE) +%s)
+
+.PHONY: debug
+debug: $(SKAFFOLD)
+	@LD_FLAGS=$(LD_FLAGS_DEBUG) $(SKAFFOLD) debug
+
+	# TODO: Andrey: P1: Inject TLS secret name dynamically into deployment
+	# GCMX_TLS_SECRET_NAME=$(kubectl -n garden get secrets | grep '^gardener-custom-metrics' | head -n 1 | awk '{print $1}') \

README.md

@@ -1,16 +1,8 @@
-pkg directory:
-```
-├── input - Takes care of source data: tracks seed's k8s contents, scrapes ShootKapis
-│   ├── controller
-│   │   ├── ...
-│   │   ├── pod
-│   │   │   └── ...
-│   │   └── secret
-│   │       └── ...
-│   ├── input_data_registry - Repository for the metrics source data
-│   │   └── input_data_registry.go
-│   └── input_data_service.go - Primary responsible for providing input data
-└── metrics_provider_service - Serves k8s metrics via HTTP
-    ├── metrics_provider.go - Implements the provider interface required by the metrics server library
-    └── metrics_provider_service.go - Primary responsible for serving K8s metrics
-```
+# gardener-custom-metrics
+
+[![REUSE status](https://api.reuse.software/badge/github.com/gardener/gardener-custom-metrics)](https://api.reuse.software/info/github.com/gardener/gardener-custom-metrics)
+
+## Overview
+
+The `gardener-custom-metrics` component operates as a K8s API service, adding functionality to the seed kube-apiserver. It periodically scrapes the metrics endpoints of all shoot kube-apiserver pods on the seed. It implements the K8s custom metrics API and provides K8s metrics specific to Gardener, based on custom calculations.
+

VERSION

@@ -1 +1 @@
-v0.10.7-dev
+v0.1.0-dev

docs/development/README.md

@@ -0,0 +1,27 @@
+> 🚧 Note: This is a WIP document.
+
+### Debugging gardener-custom-metrics
+
+1. Make sure that you have a running local Gardener setup. The steps to complete this can be found in the [Deploying Gardener Locally guide](https://github.com/gardener/gardener/blob/master/docs/deployment/getting_started_locally.md).
+
+1. In a new terminal, navigate to the gardener-custom-metrics project root.
+
+1. Make sure that your `KUBECONFIG` environment variable is targeting the local Gardener cluster.
+
+1. Run `make debug`.
+
+   This is a blocking call.  It builds and deploys a debug-instrumented pod to the cluster. It forwards the pod's log output to the console window. It forwards `localhost:56268` to the debugger port for the pod.
+
+1. Attach debugger to `localhost:56268`.
+
+   At this point, if you place a breakpoint somewhere, it should be hit.
+
+### Building and publishing gardener-custom-metrics container image:
+
+1. In a new terminal, navigate to the gardener-custom-metrics project root.
+
+1. Run `make docker-build` to build container image.
+
+1. Run `make docker-login` to authenticate against Artifact Registry before pushing the image.
+
+1. Run `make docker-push` to push the container image.

example/custom-metrics-deployment.yaml

@@ -1,31 +1,29 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: gardener-custom-metrics
+  labels:
+    app: gardener-custom-metrics
+  name: gardener-custom-metrics-debug
   namespace: garden
 spec:
   replicas: 1
   selector:
     matchLabels:
       app: gardener-custom-metrics
       gardener.cloud/role: gardener-custom-metrics
-      resources.gardener.cloud/managed-by-xxx: gardener
   template:
     metadata:
       labels:
         app: gardener-custom-metrics
         gardener.cloud/role: gardener-custom-metrics
-        resources.gardener.cloud/managed-by-xxx: gardener
         networking.gardener.cloud/from-seed: allowed
         networking.gardener.cloud/to-dns: allowed
-        networking.gardener.cloud/to-seed-apiserver: allowed
+        networking.gardener.cloud/to-runtime-apiserver: allowed
         networking.resources.gardener.cloud/to-all-shoots-kube-apiserver-tcp-443: allowed
         networking.gardener.cloud/to-apiserver: allowed
-
     spec:
       containers:
-        - command:
-            - ./gardener-custom-metrics.exe
+        - args:
             - --secure-port=6443
             - --tls-cert-file=/var/run/secrets/gardener.cloud/tls/tls.crt
             - --tls-private-key-file=/var/run/secrets/gardener.cloud/tls/tls.key
@@ -44,7 +42,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-          image: eu.gcr.io/sap-se-gcp-scp-k8s/gardener-custom-metrics/gardener-custom-metrics:v0.10.4-dev
+          image: eu.gcr.io/gardener-project/gardener/gardener-custom-metrics:v0.10.9-dev
           imagePullPolicy: IfNotPresent
           name: gardener-custom-metrics
           ports:
@@ -72,13 +70,12 @@ spec:
       securityContext:
         seccompProfile:
           type: RuntimeDefault
-      serviceAccount: gardener-custom-metrics
       serviceAccountName: gardener-custom-metrics
       terminationGracePeriodSeconds: 30
       volumes:
         - name: gardener-custom-metrics-tls
           secret:
-            secretName: gardener-custom-metrics-tls
+            secretName: gardener-custom-metrics-tls-121d8b86 # TODO: Andrey: P1: This is dynamic
         - name: kube-api-access-gardener
           projected:
             defaultMode: 420

example/custom-metrics-service.yaml

@@ -13,11 +13,10 @@ spec:
       protocol: TCP
       targetPort: 6443
   publishNotReadyAddresses: true
-  selector:
-    app: gardener-custom-metrics
-    gardener.cloud/role: gardener-custom-metrics
-    resources.gardener.cloud/managed-by-xxx: gardener
   sessionAffinity: None
   type: ClusterIP
+  # This service intentionally does not contain a pod selector. As a result, KCM does not perform any endpoint management.
+  # Endpoint management is instead done by the gardener-custom-metrics leader instance, which ensures a single endpoint,
+  # directing all traffic to the leader.
 status:
   loadBalancer: {}

example/rbac.yaml

@@ -93,7 +93,7 @@ rules:
       - ""
     resources:
       - secrets
-    # resourceNames: [ "ca", "shoot-access-prometheus" ] # TODO: Andrey: P1: How to write code so we can use name-based restriction?
+    # resourceNames: [ "ca", "shoot-access-gardener-custom-metrics" ] # TODO: Andrey: P1: How to write code so we can use name-based restriction?
     verbs:
       - get
       - list

go.mod

@@ -1,14 +1,18 @@
 module github.com/gardener/gardener-custom-metrics
 
-go 1.19
+go 1.22.0
 
 require (
 	github.com/go-logr/logr v1.2.4
+	github.com/onsi/ginkgo/v2 v2.11.0
+	github.com/onsi/gomega v1.27.8
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
+	go.uber.org/atomic v1.7.0
 	go.uber.org/zap v1.21.0
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
 	golang.org/x/time v0.3.0
+	golang.org/x/tools v0.9.3
 	k8s.io/api v0.23.17 // v0.24.0 // v0.24.4
 	k8s.io/apimachinery v0.23.17 // v0.24.0 // v0.24.4
 	k8s.io/apiserver v0.23.17 // v0.24.4
@@ -22,12 +26,6 @@ require (
 	sigs.k8s.io/metrics-server v0.6.4
 )
 
-require (
-	github.com/onsi/ginkgo/v2 v2.11.0
-	github.com/onsi/gomega v1.27.8
-	golang.org/x/tools v0.9.3
-)
-
 require (
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/PuerkitoBio/purell v1.2.0 // indirect
@@ -85,7 +83,6 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
 	go.opentelemetry.io/otel/trace v0.20.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.7.0 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.6.0 // indirect
 	golang.org/x/mod v0.10.0 // indirect